@dxos/client-services 0.1.56-main.bf228a7 → 0.1.56-main.c19b7cd

package/src/packlets/identity/identity.test.ts

@@ -4,6 +4,7 @@
 
 import expect from 'expect';
 
+import { Context } from '@dxos/context';
 import { CredentialGenerator, verifyCredential } from '@dxos/credentials';
 import {
   MetadataStore,
@@ -106,8 +107,8 @@ describe('identity/identity', () => {
       space,
     });
 
-    await identity.open();
-    afterTest(() => identity.close());
+    await identity.open(new Context());
+    afterTest(() => identity.close(new Context()));
 
     //
     // Identity genesis
@@ -219,8 +220,8 @@ describe('identity/identity', () => {
         space,
       }));
 
-      await identity.open();
-      afterTest(() => identity.close());
+      await identity.open(new Context());
+      afterTest(() => identity.close(new Context()));
 
       //
       // Identity genesis
@@ -310,8 +311,8 @@ describe('identity/identity', () => {
         space,
       }));
 
-      await identity.open();
-      afterTest(() => identity.close());
+      await identity.open(new Context());
+      afterTest(() => identity.close(new Context()));
     }
 
     //

package/src/packlets/identity/identity.ts

@@ -4,6 +4,7 @@
 
 import { Event } from '@dxos/async';
 import { AUTH_TIMEOUT, LOAD_CONTROL_FEEDS_TIMEOUT } from '@dxos/client-protocol';
+import { Context } from '@dxos/context';
 import {
   DeviceStateMachine,
   CredentialSigner,
@@ -21,6 +22,7 @@ import { log } from '@dxos/log';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { DeviceAdmissionRequest } from '@dxos/protocols/proto/dxos/halo/invitations';
+import { trace } from '@dxos/tracing';
 import { ComplexSet } from '@dxos/util';
 
 import { TrustedKeySetAuthVerifier } from './authenticator';
@@ -35,6 +37,7 @@ export type IdentityParams = {
 /**
  * Agent identity manager, which includes the agent's Halo space.
  */
+@trace.resource()
 export class Identity {
   public readonly space: Space;
   private readonly _signer: Signer;
@@ -76,13 +79,15 @@ export class Identity {
     return this._deviceStateMachine.authorizedDeviceKeys;
   }
 
-  async open() {
+  @trace.span()
+  async open(ctx: Context) {
     await this.space.spaceState.addCredentialProcessor(this._deviceStateMachine);
     await this.space.spaceState.addCredentialProcessor(this._profileStateMachine);
-    await this.space.open();
+    await this.space.open(ctx);
   }
 
-  async close() {
+  @trace.span()
+  async close(ctx: Context) {
     await this.authVerifier.close();
     await this.space.spaceState.removeCredentialProcessor(this._profileStateMachine);
     await this.space.spaceState.removeCredentialProcessor(this._deviceStateMachine);

package/src/packlets/invitations/device-invitation-protocol.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { asyncChain } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test, afterTest } from '@dxos/test';
 
@@ -19,7 +20,7 @@ const closeAfterTest = async (peer: ServiceContext) => {
 describe('services/device', () => {
   test('creates identity', async () => {
     const peer = createServiceContext();
-    await peer.open();
+    await peer.open(new Context());
     afterTest(() => peer.close());
 
     const identity = await peer.createIdentity();

package/src/packlets/logging/logging-service.ts

@@ -13,6 +13,7 @@ import {
   ControlMetricsRequest,
   ControlMetricsResponse,
   QueryMetricsRequest,
+  QueryMetricsResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { jsonify, numericalValues, tracer } from '@dxos/util';
 
@@ -21,6 +22,7 @@ import { jsonify, numericalValues, tracer } from '@dxos/util';
  */
 export class LoggingServiceImpl implements LoggingService {
   private readonly _logs = new Event<NaturalLogEntry>();
+  private readonly _started = new Date();
 
   async open() {
     log.runtimeConfig.processors.push(this._logProcessor);
@@ -45,11 +47,14 @@ export class LoggingServiceImpl implements LoggingService {
     return { recording: tracer.recording };
   }
 
-  queryMetrics({ interval }: QueryMetricsRequest): Stream<Metrics> {
+  /**
+   * @deprecated (Move to diagnostics).
+   */
+  queryMetrics({ interval = 5_000 }: QueryMetricsRequest): Stream<QueryMetricsResponse> {
     // TODO(burdon): Map all traces; how to bind to reducer/metrics shape (e.g., numericalValues)?
-    const createNumericalValues = (key: string) => {
-      const consume = tracer.get(key) ?? [];
-      return { key, stats: numericalValues(consume, 'duration') };
+    const getNumericalValues = (key: string) => {
+      const events = tracer.get(key) ?? [];
+      return { key, stats: numericalValues(events, 'duration') };
     };
 
     return new Stream(({ next }) => {
@@ -57,16 +62,19 @@ export class LoggingServiceImpl implements LoggingService {
         const metrics: Metrics = {
           timestamp: new Date(),
           values: [
-            createNumericalValues('dxos.echo.pipeline.control'),
-            createNumericalValues('dxos.echo.pipeline.data'),
-          ],
+            getNumericalValues('dxos.echo.pipeline.control'),
+            getNumericalValues('dxos.echo.pipeline.data'),
+          ].filter(Boolean) as Metrics.KeyPair[],
         };
 
-        next(metrics);
+        next({
+          timestamp: new Date(),
+          metrics,
+        });
       };
 
       update();
-      const i = setInterval(update, interval);
+      const i = setInterval(update, Math.max(interval, 1_000));
       return () => {
         clearInterval(i);
       };

package/src/packlets/network/network-service.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { NetworkService, ConnectionState } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
 
@@ -18,7 +19,7 @@ describe('NetworkService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     networkService = new NetworkServiceImpl(serviceContext.networkManager, serviceContext.signalManager);
   });
 

package/src/packlets/services/client-rpc-server.ts

@@ -5,7 +5,7 @@
 import { ClientServices } from '@dxos/client-protocol';
 import { Any, ServiceHandler, Stream } from '@dxos/codec-protobuf';
 import { raise } from '@dxos/debug';
-import { parseMethodName, RpcPeer, RpcPeerOptions } from '@dxos/rpc';
+import { parseMethodName, RpcPeer, RpcPeerOptions, ServiceBundle } from '@dxos/rpc';
 import { MaybePromise } from '@dxos/util';
 
 import { ServiceRegistry } from './service-registry';
@@ -74,16 +74,16 @@ export class ClientRpcServer {
   private _getServiceHandler(serviceName: string) {
     if (!this._handlerCache.has(serviceName)) {
       const [key, descriptor] =
-        Object.entries(this._serviceRegistry.descriptors).find(
+        Object.entries(this._serviceRegistry.descriptors as ServiceBundle<Record<string, any>>).find(
           ([key, descriptor]) => descriptor.name === serviceName,
         ) ?? raise(new Error(`Service not available: ${serviceName}`));
 
-      const service = this._serviceRegistry.services[key as keyof ClientServices];
+      const service = this._serviceRegistry.services[key as keyof ClientServices] as any;
       if (!service) {
         throw new Error(`Service not available: ${serviceName}`);
       }
 
-      this._handlerCache.set(serviceName, descriptor.createServer(service as any));
+      this._handlerCache.set(serviceName, descriptor.createServer(service));
     }
 
     return this._handlerCache.get(serviceName)!;

package/src/packlets/services/diagnostics.ts

@@ -0,0 +1,211 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { ClientServices } from '@dxos/client-protocol';
+import { getFirstStreamValue } from '@dxos/codec-protobuf';
+import { Config, ConfigProto } from '@dxos/config';
+import { credentialTypeFilter } from '@dxos/credentials';
+import { DocumentModel, DocumentModelState } from '@dxos/document-model';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { STORAGE_VERSION } from '@dxos/protocols';
+import {
+  Device,
+  Identity,
+  Metrics,
+  Space as SpaceProto,
+  SpaceMember,
+} from '@dxos/protocols/proto/dxos/client/services';
+import { SubscribeToFeedsResponse } from '@dxos/protocols/proto/dxos/devtools/host';
+import { Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
+
+import { DXOS_VERSION } from '../../version';
+import { DataSpace } from '../spaces';
+import { getPlatform, Platform } from './platform';
+import { ServiceContext } from './service-context';
+
+const DEFAULT_TIMEOUT = 1_000;
+
+export type Diagnostics = {
+  created: string;
+  platform: Platform;
+  config?: ConfigProto;
+  client: {
+    version: string;
+    storage: {
+      version: number;
+    };
+  };
+  identity?: Identity;
+  devices?: Device[];
+  spaces?: SpaceStats[];
+  feeds?: Partial<SubscribeToFeedsResponse.Feed>[];
+  metrics?: Metrics;
+};
+
+// TODO(burdon): Normalize for ECHO/HALO.
+export type SpaceStats = {
+  key: PublicKey;
+  properties?: {
+    name: string;
+  };
+  db?: {
+    objects: number;
+  };
+  metrics?: SpaceProto.Metrics & {
+    startupTime?: number;
+  };
+  epochs?: (Epoch & { id?: PublicKey })[];
+  members?: SpaceMember[];
+  pipeline?: SpaceProto.PipelineState;
+};
+
+/**
+ * Create diagnostics to provide snapshot of current system state.
+ */
+export const createDiagnostics = async (
+  clientServices: Partial<ClientServices>,
+  serviceContext: ServiceContext,
+  config: Config,
+): Promise<Diagnostics> => {
+  const diagnostics: Diagnostics = {
+    created: new Date().toISOString(),
+    platform: getPlatform(),
+    client: {
+      version: DXOS_VERSION,
+      storage: {
+        version: STORAGE_VERSION,
+      },
+    },
+  };
+
+  // Trace metrics.
+  // TODO(burdon): Move here from logging service?
+  {
+    invariant(clientServices.LoggingService, 'SystemService is not available.');
+    diagnostics.metrics = await getFirstStreamValue(clientServices.LoggingService.queryMetrics({}), {
+      timeout: DEFAULT_TIMEOUT,
+    }).catch(() => undefined);
+  }
+
+  const identity = serviceContext.identityManager.identity;
+  if (identity) {
+    // Identity.
+    diagnostics.identity = {
+      identityKey: identity.identityKey,
+      spaceKey: identity.space.key,
+      profile: identity.profileDocument,
+    };
+
+    // Devices.
+    const { devices } =
+      (await getFirstStreamValue(clientServices.DevicesService!.queryDevices(), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.devices = devices;
+
+    // TODO(dmaretskyi): Add metrics for halo space.
+
+    // Spaces.
+    if (serviceContext.dataSpaceManager) {
+      diagnostics.spaces = await Promise.all(
+        Array.from(serviceContext.dataSpaceManager.spaces.values()).map((space) => getSpaceStats(space)) ?? [],
+      );
+    }
+
+    // Feeds.
+    const { feeds = [] } =
+      (await getFirstStreamValue(clientServices.DevtoolsHost!.subscribeToFeeds({}), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.feeds = feeds.map(({ feedKey, bytes, length }) => ({ feedKey, bytes, length }));
+  }
+
+  diagnostics.config = config.values;
+
+  return diagnostics;
+};
+
+const getProperties = (space: DataSpace) => {
+  let properties: any = {};
+  try {
+    // Add properties to cache.
+    const propertiesItem = space.dataPipeline.itemManager.items.find(
+      (item) =>
+        item.modelMeta?.type === DocumentModel.meta.type &&
+        (item.state as DocumentModelState)?.type === 'dxos.sdk.client.Properties',
+    );
+
+    const state = propertiesItem?.state as DocumentModelState;
+    properties = state?.data;
+  } catch (err: any) {
+    log.warn(err.message);
+  }
+
+  return properties;
+};
+
+const getSpaceStats = async (space: DataSpace): Promise<SpaceStats> => {
+  const stats: SpaceStats = {
+    key: space.key,
+    metrics: space.metrics,
+
+    epochs: space.inner.spaceState.credentials
+      .filter(credentialTypeFilter('dxos.halo.credentials.Epoch'))
+      .map((credential) => ({
+        ...credential.subject.assertion,
+        id: credential.id,
+      })),
+
+    members: Array.from(space.inner.spaceState.members.values()).map((member) => ({
+      identity: {
+        identityKey: member.key,
+        profile: {
+          displayName: member.assertion.profile?.displayName,
+        },
+      },
+      presence:
+        space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key)).length > 0
+          ? SpaceMember.PresenceState.ONLINE
+          : SpaceMember.PresenceState.OFFLINE,
+    })),
+
+    pipeline: {
+      // TODO(burdon): Pick properties from credentials if needed.
+      // currentEpoch: space.dataPipeline.currentEpoch,
+      // appliedEpoch: space.dataPipeline.appliedEpoch,
+
+      controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
+      currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
+      targetControlTimeframe: space.inner.controlPipeline.state.targetTimeframe,
+      totalControlTimeframe: space.inner.controlPipeline.state.endTimeframe,
+
+      // TODO(burdon): Empty?
+      dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
+      startDataTimeframe: space.dataPipeline.pipelineState?.startTimeframe,
+      currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
+      targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
+      totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
+    },
+  };
+
+  // TODO(burdon): May not be open?
+  if (space.dataPipeline.itemManager) {
+    Object.assign(stats, {
+      properties: getProperties(space),
+      db: {
+        objects: space.dataPipeline.itemManager.entities.size,
+      },
+    } as SpaceStats);
+  }
+
+  // TODO(burdon): Factor out.
+  if (stats.metrics) {
+    const { open, ready } = stats.metrics;
+    stats.metrics.startupTime = open && ready && ready.getTime() - open.getTime();
+  }
+
+  return stats;
+};

package/src/packlets/services/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './client-rpc-server';
+export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';

package/src/packlets/services/platform.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export type Platform = {
+  type: 'browser' | 'shared-worker' | 'node';
+  userAgent?: string;
+  platform?: string;
+  runtime?: string;
+  uptime?: number;
+  memory?: {
+    rss: number;
+    heapTotal: number;
+    heapUsed: number;
+    external: number;
+    arrayBuffers: number;
+  };
+};
+
+export const getPlatform = (): Platform => {
+  if ((process as any).browser) {
+    if (typeof window !== 'undefined') {
+      // Browser.
+      const { userAgent } = window.navigator;
+      return {
+        type: 'browser',
+        userAgent,
+        uptime: Math.floor((Date.now() - window.performance.timeOrigin) / 1_000),
+      };
+    } else {
+      // Shared worker.
+      return {
+        type: 'shared-worker',
+        uptime: Math.floor((Date.now() - performance.timeOrigin) / 1_000),
+      };
+    }
+  } else {
+    // Node.
+    const { platform, version, arch } = process;
+    return {
+      type: 'node',
+      platform: `${platform} ${version} ${arch}`,
+      runtime: process.version,
+      uptime: Math.floor(process.uptime()),
+      memory: process.memoryUsage(),
+    };
+  }
+};

package/src/packlets/services/service-context.test.ts

@@ -2,9 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
+import { expect } from 'chai';
+
+import { DocumentModel, DocumentModelState, MutationBuilder } from '@dxos/document-model';
+import { createModelMutation, encodeModelMutation, genesisMutation } from '@dxos/echo-db';
+import { WriteReceipt } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
 
 import { createServiceContext, syncItemsLocal } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -16,11 +25,61 @@ describe('services/ServiceContext', () => {
     await device1.createIdentity();
     const space1 = await device1.dataSpaceManager!.createSpace();
 
+    const itemId = PublicKey.random().toHex();
+    await space1.dataPipeline.databaseHost!.getWriteStream()?.write({
+      batch: genesisMutation(itemId, DocumentModel.meta.type),
+    });
+
+    let counter = 0;
+
+    for (const _ in range(5)) {
+      const receipts: WriteReceipt[] = [];
+      for (const _ in range(50)) {
+        receipts.push(
+          await space1.dataPipeline.databaseHost!.getWriteStream()!.write({
+            batch: createModelMutation(
+              itemId,
+              encodeModelMutation(DocumentModel.meta, new MutationBuilder().set('counter', ++counter).build()),
+            ),
+          }),
+        );
+      }
+
+      await space1.dataPipeline.pipelineState!.waitUntilTimeframe(
+        Timeframe.merge(...receipts.map((receipt) => new Timeframe([[receipt.feedKey, receipt.seq]]))),
+      );
+
+      await space1.createEpoch();
+      log.info('epoch', { number: space1.dataPipeline.currentEpoch?.subject.assertion.number });
+    }
+
     const device2 = createServiceContext({ signalContext: networkContext });
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     await device2.dataSpaceManager!.waitUntilSpaceReady(space1!.key);
     const space2 = await device2.dataSpaceManager!.spaces.get(space1.key);
+
+    log.info('peer 2', {
+      currentEpoch: space2!.dataPipeline.currentEpoch?.subject.assertion.number,
+      appliedEpoch: space2!.dataPipeline.appliedEpoch?.subject.assertion.number,
+    });
+
+    await space2?.dataPipeline.onNewEpoch.waitForCondition(
+      () =>
+        space2!.dataPipeline.appliedEpoch?.subject.assertion.number ===
+        space1.dataPipeline.currentEpoch?.subject.assertion.number,
+    );
+
+    expect(space2!.dataPipeline.currentEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.currentEpoch!.subject.assertion.number,
+    );
+    expect(space2!.dataPipeline.appliedEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.appliedEpoch!.subject.assertion.number,
+    );
+
+    const item2 = space2!.dataPipeline.itemManager.entities.get(itemId);
+    expect((item2!.state as DocumentModelState).data.counter).to.equal(counter);
+
     await syncItemsLocal(space1.dataPipeline, space2!.dataPipeline);
   });
 

package/src/packlets/services/service-context.ts

@@ -3,6 +3,7 @@
 //
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { CredentialProcessor, getCredentialAssertion } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import {
@@ -26,6 +27,7 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Storage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { trace as Trace } from '@dxos/tracing';
 
 import { CreateIdentityOptions, IdentityManager, JoinIdentityParams } from '../identity';
 import {
@@ -40,6 +42,7 @@ import { DataSpaceManager, SigningContext } from '../spaces';
  * Shared backend for all client services.
  */
 // TODO(burdon): Rename/break-up into smaller components. And/or make members private.
+@Trace.resource()
 export class ServiceContext {
   public readonly initialized = new Trigger();
   public readonly dataServiceSubscriptions = new DataServiceSubscriptions();
@@ -121,21 +124,21 @@ export class ServiceContext {
     );
   }
 
-  async open() {
-    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
-
+  @Trace.span()
+  async open(ctx: Context) {
     await this._checkStorageVersion();
 
     log('opening...');
+    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
     await this.signalManager.open();
     await this.networkManager.open();
     await this.spaceManager.open();
-    await this.identityManager.open();
+    await this.identityManager.open(ctx);
     if (this.identityManager.identity) {
-      await this._initialize();
+      await this._initialize(ctx);
     }
-    log('opened');
     log.trace('dxos.sdk.service-context.open', trace.end({ id: this._instanceId }));
+    log('opened');
   }
 
   async close() {
@@ -156,7 +159,7 @@ export class ServiceContext {
   async createIdentity(params: CreateIdentityOptions = {}) {
     const identity = await this.identityManager.createIdentity(params);
 
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -169,7 +172,7 @@ export class ServiceContext {
   private async _acceptIdentity(params: JoinIdentityParams) {
     const identity = await this.identityManager.acceptIdentity(params);
 
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -182,7 +185,8 @@ export class ServiceContext {
   }
 
   // Called when identity is created.
-  private async _initialize() {
+  @Trace.span()
+  private async _initialize(ctx: Context) {
     log('initializing spaces...');
     const identity = this.identityManager.identity ?? failUndefined();
     const signingContext: SigningContext = {