@dxos/client-services 0.1.56-main.83e9b8a → 0.1.56-main.a25bb93

package/src/packlets/logging/logging-service.ts

@@ -13,6 +13,7 @@ import {
   ControlMetricsRequest,
   ControlMetricsResponse,
   QueryMetricsRequest,
+  QueryMetricsResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { jsonify, numericalValues, tracer } from '@dxos/util';
 
@@ -21,6 +22,7 @@ import { jsonify, numericalValues, tracer } from '@dxos/util';
  */
 export class LoggingServiceImpl implements LoggingService {
   private readonly _logs = new Event<NaturalLogEntry>();
+  private readonly _started = new Date();
 
   async open() {
     log.runtimeConfig.processors.push(this._logProcessor);
@@ -45,11 +47,14 @@ export class LoggingServiceImpl implements LoggingService {
     return { recording: tracer.recording };
   }
 
-  queryMetrics({ interval }: QueryMetricsRequest): Stream<Metrics> {
+  /**
+   * @deprecated (Move to diagnostics).
+   */
+  queryMetrics({ interval = 5_000 }: QueryMetricsRequest): Stream<QueryMetricsResponse> {
     // TODO(burdon): Map all traces; how to bind to reducer/metrics shape (e.g., numericalValues)?
-    const createNumericalValues = (key: string) => {
-      const consume = tracer.get(key) ?? [];
-      return { key, stats: numericalValues(consume, 'duration') };
+    const getNumericalValues = (key: string) => {
+      const events = tracer.get(key) ?? [];
+      return { key, stats: numericalValues(events, 'duration') };
     };
 
     return new Stream(({ next }) => {
@@ -57,16 +62,19 @@ export class LoggingServiceImpl implements LoggingService {
         const metrics: Metrics = {
           timestamp: new Date(),
           values: [
-            createNumericalValues('dxos.echo.pipeline.control'),
-            createNumericalValues('dxos.echo.pipeline.data'),
-          ],
+            getNumericalValues('dxos.echo.pipeline.control'),
+            getNumericalValues('dxos.echo.pipeline.data'),
+          ].filter(Boolean) as Metrics.KeyPair[],
         };
 
-        next(metrics);
+        next({
+          timestamp: new Date(),
+          metrics,
+        });
       };
 
       update();
-      const i = setInterval(update, interval);
+      const i = setInterval(update, Math.max(interval, 1_000));
       return () => {
         clearInterval(i);
       };

package/src/packlets/services/diagnostics.ts

@@ -0,0 +1,211 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { ClientServices } from '@dxos/client-protocol';
+import { getFirstStreamValue } from '@dxos/codec-protobuf';
+import { Config, ConfigProto } from '@dxos/config';
+import { credentialTypeFilter } from '@dxos/credentials';
+import { DocumentModel, DocumentModelState } from '@dxos/document-model';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { STORAGE_VERSION } from '@dxos/protocols';
+import {
+  Device,
+  Identity,
+  Metrics,
+  Space as SpaceProto,
+  SpaceMember,
+} from '@dxos/protocols/proto/dxos/client/services';
+import { SubscribeToFeedsResponse } from '@dxos/protocols/proto/dxos/devtools/host';
+import { Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
+
+import { DXOS_VERSION } from '../../version';
+import { DataSpace } from '../spaces';
+import { getPlatform, Platform } from './platform';
+import { ServiceContext } from './service-context';
+
+const DEFAULT_TIMEOUT = 1_000;
+
+export type Diagnostics = {
+  created: string;
+  platform: Platform;
+  config?: ConfigProto;
+  client: {
+    version: string;
+    storage: {
+      version: number;
+    };
+  };
+  identity?: Identity;
+  devices?: Device[];
+  spaces?: SpaceStats[];
+  feeds?: Partial<SubscribeToFeedsResponse.Feed>[];
+  metrics?: Metrics;
+};
+
+// TODO(burdon): Normalize for ECHO/HALO.
+export type SpaceStats = {
+  key: PublicKey;
+  properties?: {
+    name: string;
+  };
+  db?: {
+    objects: number;
+  };
+  metrics?: SpaceProto.Metrics & {
+    startupTime?: number;
+  };
+  epochs?: (Epoch & { id?: PublicKey })[];
+  members?: SpaceMember[];
+  pipeline?: SpaceProto.PipelineState;
+};
+
+/**
+ * Create diagnostics to provide snapshot of current system state.
+ */
+export const createDiagnostics = async (
+  clientServices: Partial<ClientServices>,
+  serviceContext: ServiceContext,
+  config: Config,
+): Promise<Diagnostics> => {
+  const diagnostics: Diagnostics = {
+    created: new Date().toISOString(),
+    platform: getPlatform(),
+    client: {
+      version: DXOS_VERSION,
+      storage: {
+        version: STORAGE_VERSION,
+      },
+    },
+  };
+
+  // Trace metrics.
+  // TODO(burdon): Move here from logging service?
+  {
+    invariant(clientServices.LoggingService, 'SystemService is not available.');
+    diagnostics.metrics = await getFirstStreamValue(clientServices.LoggingService.queryMetrics({}), {
+      timeout: DEFAULT_TIMEOUT,
+    }).catch(() => undefined);
+  }
+
+  const identity = serviceContext.identityManager.identity;
+  if (identity) {
+    // Identity.
+    diagnostics.identity = {
+      identityKey: identity.identityKey,
+      spaceKey: identity.space.key,
+      profile: identity.profileDocument,
+    };
+
+    // Devices.
+    const { devices } =
+      (await getFirstStreamValue(clientServices.DevicesService!.queryDevices(), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.devices = devices;
+
+    // TODO(dmaretskyi): Add metrics for halo space.
+
+    // Spaces.
+    if (serviceContext.dataSpaceManager) {
+      diagnostics.spaces = await Promise.all(
+        Array.from(serviceContext.dataSpaceManager.spaces.values()).map((space) => getSpaceStats(space)) ?? [],
+      );
+    }
+
+    // Feeds.
+    const { feeds = [] } =
+      (await getFirstStreamValue(clientServices.DevtoolsHost!.subscribeToFeeds({}), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.feeds = feeds.map(({ feedKey, bytes, length }) => ({ feedKey, bytes, length }));
+  }
+
+  diagnostics.config = config.values;
+
+  return diagnostics;
+};
+
+const getProperties = (space: DataSpace) => {
+  let properties: any = {};
+  try {
+    // Add properties to cache.
+    const propertiesItem = space.dataPipeline.itemManager.items.find(
+      (item) =>
+        item.modelMeta?.type === DocumentModel.meta.type &&
+        (item.state as DocumentModelState)?.type === 'dxos.sdk.client.Properties',
+    );
+
+    const state = propertiesItem?.state as DocumentModelState;
+    properties = state?.data;
+  } catch (err: any) {
+    log.warn(err.message);
+  }
+
+  return properties;
+};
+
+const getSpaceStats = async (space: DataSpace): Promise<SpaceStats> => {
+  const stats: SpaceStats = {
+    key: space.key,
+    metrics: space.metrics,
+
+    epochs: space.inner.spaceState.credentials
+      .filter(credentialTypeFilter('dxos.halo.credentials.Epoch'))
+      .map((credential) => ({
+        ...credential.subject.assertion,
+        id: credential.id,
+      })),
+
+    members: Array.from(space.inner.spaceState.members.values()).map((member) => ({
+      identity: {
+        identityKey: member.key,
+        profile: {
+          displayName: member.assertion.profile?.displayName,
+        },
+      },
+      presence:
+        space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key)).length > 0
+          ? SpaceMember.PresenceState.ONLINE
+          : SpaceMember.PresenceState.OFFLINE,
+    })),
+
+    pipeline: {
+      // TODO(burdon): Pick properties from credentials if needed.
+      // currentEpoch: space.dataPipeline.currentEpoch,
+      // appliedEpoch: space.dataPipeline.appliedEpoch,
+
+      controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
+      currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
+      targetControlTimeframe: space.inner.controlPipeline.state.targetTimeframe,
+      totalControlTimeframe: space.inner.controlPipeline.state.endTimeframe,
+
+      // TODO(burdon): Empty?
+      dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
+      startDataTimeframe: space.dataPipeline.pipelineState?.startTimeframe,
+      currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
+      targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
+      totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
+    },
+  };
+
+  // TODO(burdon): May not be open?
+  if (space.dataPipeline.itemManager) {
+    Object.assign(stats, {
+      properties: getProperties(space),
+      db: {
+        objects: space.dataPipeline.itemManager.entities.size,
+      },
+    } as SpaceStats);
+  }
+
+  // TODO(burdon): Factor out.
+  if (stats.metrics) {
+    const { open, ready } = stats.metrics;
+    stats.metrics.startupTime = open && ready && ready.getTime() - open.getTime();
+  }
+
+  return stats;
+};

package/src/packlets/services/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './client-rpc-server';
+export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';

package/src/packlets/services/platform.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export type Platform = {
+  type: 'browser' | 'shared-worker' | 'node';
+  userAgent?: string;
+  platform?: string;
+  runtime?: string;
+  uptime?: number;
+  memory?: {
+    rss: number;
+    heapTotal: number;
+    heapUsed: number;
+    external: number;
+    arrayBuffers: number;
+  };
+};
+
+export const getPlatform = (): Platform => {
+  if ((process as any).browser) {
+    if (typeof window !== 'undefined') {
+      // Browser.
+      const { userAgent } = window.navigator;
+      return {
+        type: 'browser',
+        userAgent,
+        uptime: Math.floor((Date.now() - window.performance.timeOrigin) / 1_000),
+      };
+    } else {
+      // Shared worker.
+      return {
+        type: 'shared-worker',
+        uptime: Math.floor((Date.now() - performance.timeOrigin) / 1_000),
+      };
+    }
+  } else {
+    // Node.
+    const { platform, version, arch } = process;
+    return {
+      type: 'node',
+      platform: `${platform} ${version} ${arch}`,
+      runtime: process.version,
+      uptime: Math.floor(process.uptime()),
+      memory: process.memoryUsage(),
+    };
+  }
+};

package/src/packlets/services/service-context.ts

@@ -2,8 +2,6 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Trigger } from '@dxos/async';
 import { CredentialProcessor, getCredentialAssertion } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
@@ -15,6 +13,7 @@ import {
   SnapshotStore,
 } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -123,11 +122,10 @@ export class ServiceContext {
   }
 
   async open() {
-    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
-
     await this._checkStorageVersion();
 
     log('opening...');
+    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
     await this.signalManager.open();
     await this.networkManager.open();
     await this.spaceManager.open();
@@ -135,8 +133,8 @@ export class ServiceContext {
     if (this.identityManager.identity) {
       await this._initialize();
     }
-    log('opened');
     log.trace('dxos.sdk.service-context.open', trace.end({ id: this._instanceId }));
+    log('opened');
   }
 
   async close() {

package/src/packlets/services/service-host.test.ts

@@ -55,8 +55,8 @@ describe('ClientServicesHost', () => {
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     // Test if Identity exposes haloSpace key.
@@ -92,8 +92,8 @@ describe('ClientServicesHost', () => {
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     const nonce = new Uint8Array([0, 0, 0, 0]);
@@ -121,7 +121,7 @@ describe('ClientServicesHost', () => {
 
       await host.services.IdentityService?.createIdentity({});
 
-      expect(host._serviceContext.storage.size).to.exist;
+      expect(host.context.storage.size).to.exist;
 
       await asyncTimeout(host.reset(), 1000);
       await host.close();

package/src/packlets/services/service-host.ts

@@ -2,13 +2,12 @@
 // Copyright 2021 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, ClientServices } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
 import { DocumentModel } from '@dxos/document-model';
 import { DataServiceImpl } from '@dxos/echo-pipeline';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { SignalManager, WebsocketSignalManager } from '@dxos/messaging';
@@ -29,6 +28,7 @@ import { NetworkServiceImpl } from '../network';
 import { SpacesServiceImpl } from '../spaces';
 import { createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
+import { createDiagnostics } from './diagnostics';
 import { ServiceContext } from './service-context';
 import { ServiceRegistry } from './service-registry';
 
@@ -79,17 +79,15 @@ export class ClientServicesHost {
   private _storage?: Storage;
   private _callbacks?: ClientServicesHostCallbacks;
 
-  _serviceContext!: ServiceContext;
+  private _serviceContext!: ServiceContext;
   private _opening = false;
   private _open = false;
 
   constructor({
     config,
     modelFactory = createDefaultModelFactory(),
-    // TODO(burdon): Create ApolloLink abstraction (see Client).
     transportFactory,
     signalManager,
-    connectionLog,
     storage,
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
@@ -117,11 +115,11 @@ export class ClientServicesHost {
 
     this._systemService = new SystemServiceImpl({
       config: this._config,
-
       statusUpdate: this._statusUpdate,
-
       getCurrentStatus: () => (this.isOpen ? SystemStatus.ACTIVE : SystemStatus.INACTIVE),
-
+      getDiagnostics: () => {
+        return createDiagnostics(this._serviceRegistry.services, this._serviceContext, this._config!);
+      },
       onUpdateStatus: async (status: SystemStatus) => {
         if (!this.isOpen && status === SystemStatus.ACTIVE) {
           await this._resourceLock?.acquire();
@@ -129,7 +127,6 @@ export class ClientServicesHost {
           await this._resourceLock?.release();
         }
       },
-
       onReset: async () => {
         await this.reset();
       },
@@ -137,7 +134,6 @@ export class ClientServicesHost {
 
     this._loggingService = new LoggingServiceImpl();
 
-    // TODO(burdon): Start to think of DMG (dynamic services).
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
       SystemService: this._systemService,
     });
@@ -147,6 +143,14 @@ export class ClientServicesHost {
     return this._open;
   }
 
+  get config() {
+    return this._config;
+  }
+
+  get context() {
+    return this._serviceContext;
+  }
+
   get serviceRegistry() {
     return this._serviceRegistry;
   }
@@ -166,10 +170,10 @@ export class ClientServicesHost {
    */
   initialize({ config, ...options }: InitializeOptions) {
     invariant(!this._open, 'service host is open');
+    log.info('initializing...');
 
     if (config) {
       invariant(!this._config, 'config already set');
-
       this._config = config;
       if (!this._storage) {
         this._storage = createStorageObjects(config.get('runtime.client.storage', {})!).storage;
@@ -191,6 +195,8 @@ export class ClientServicesHost {
       transportFactory,
       signalManager,
     });
+
+    log.info('initialized');
   }
 
   @synchronized
@@ -200,7 +206,7 @@ export class ClientServicesHost {
     }
 
     const traceId = PublicKey.random().toHex();
-    log.trace('dxos.sdk.client-services-host.open', trace.begin({ id: traceId }));
+    log.trace('dxos.client-services.host.open', trace.begin({ id: traceId }));
 
     invariant(this._config, 'config not set');
     invariant(this._storage, 'storage not set');
@@ -208,13 +214,11 @@ export class ClientServicesHost {
     invariant(this._networkManager, 'network manager not set');
 
     this._opening = true;
-    log('opening...', { lockKey: this._resourceLock?.lockKey });
+    log.info('opening...', { lockKey: this._resourceLock?.lockKey });
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
 
-    // TODO(wittjosiah): Make re-entrant.
-    // TODO(burdon): Break into components.
     this._serviceContext = new ServiceContext(
       this._storage,
       this._networkManager,
@@ -222,7 +226,6 @@ export class ClientServicesHost {
       this._modelFactory,
     );
 
-    // TODO(burdon): Start to think of DMG (dynamic services).
     this._serviceRegistry.setServices({
       SystemService: this._systemService,
 
@@ -263,8 +266,8 @@ export class ClientServicesHost {
     this._open = true;
     this._statusUpdate.emit();
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log('opened', { deviceKey });
-    log.trace('dxos.sdk.client-services-host.open', trace.end({ id: traceId }));
+    log.info('opened', { deviceKey });
+    log.trace('dxos.client-services.host.open', trace.end({ id: traceId }));
   }
 
   @synchronized
@@ -274,13 +277,13 @@ export class ClientServicesHost {
     }
 
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log('closing...', { deviceKey });
+    log.info('closing...', { deviceKey });
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
     await this._serviceContext.close();
     this._open = false;
     this._statusUpdate.emit();
-    log('closed', { deviceKey });
+    log.info('closed', { deviceKey });
   }
 
   async reset() {

package/src/packlets/services/service-registry.test.ts

@@ -37,6 +37,7 @@ describe('service registry', () => {
       SystemService: new SystemServiceImpl({
         config: new Config({ runtime: { client: { remoteSource } } }),
         getCurrentStatus: () => SystemStatus.ACTIVE,
+        getDiagnostics: async () => ({}),
         onReset: () => {},
         onUpdateStatus: () => {},
         statusUpdate: new Event(),

package/src/packlets/services/util.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { PublicKey } from '@dxos/keys';
+import { humanize } from '@dxos/util';
+
+// TODO(burdon): Move to util.
+
+export type JsonStringifyOptions = {
+  truncate?: boolean;
+  humanize?: boolean;
+};
+
+export const jsonStringify = <T>(data: T, options: JsonStringifyOptions) => {
+  if (options.humanize || options.truncate) {
+    return JSON.parse(JSON.stringify(data, jsonKeyReplacer(options)));
+  }
+
+  return data;
+};
+
+const jsonKeyReplacer = (options: JsonStringifyOptions) => (key: string, value: any) => {
+  if (typeof value === 'string') {
+    // Infer if key.
+    const key = PublicKey.fromHex(value);
+    if (key.toHex() === value) {
+      return options.humanize ? humanize(key) : key.truncate();
+    }
+  }
+
+  return value;
+};

package/src/packlets/spaces/data-space-manager.ts

@@ -2,13 +2,12 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, synchronized, trackLeaks } from '@dxos/async';
 import { cancelWithContext, Context } from '@dxos/context';
 import { CredentialSigner, getCredentialAssertion } from '@dxos/credentials';
 import { DataServiceSubscriptions, MetadataStore, Space, SpaceManager } from '@dxos/echo-pipeline';
 import { FeedStore } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';

package/src/packlets/spaces/notarization-plugin.ts

@@ -2,12 +2,11 @@
 // Copyright 2023 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { DeferredTask, Event, scheduleTask, sleep, TimeoutError, Trigger } from '@dxos/async';
 import { Context, rejectOnDispose } from '@dxos/context';
 import { CredentialProcessor } from '@dxos/credentials';
 import { FeedWriter } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols';

package/src/packlets/system/system-service.test.ts

@@ -35,6 +35,7 @@ describe('SystemService', () => {
       config,
       statusUpdate,
       getCurrentStatus: () => currentStatus,
+      getDiagnostics: async () => ({}),
       onUpdateStatus: (status) => {
         updateStatus.wake(status);
       },