@dxos/client-services 0.1.56-main.09ca29d → 0.1.56-main.0e9f69e

package/src/packlets/identity/identity-manager.ts

@@ -3,6 +3,7 @@
 //
 
 import { Event } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { createCredentialSignerWithKey, CredentialGenerator } from '@dxos/credentials';
 import { MetadataStore, SpaceManager, SwarmIdentity } from '@dxos/echo-pipeline';
 import { FeedStore } from '@dxos/feed-store';
@@ -13,8 +14,9 @@ import { log } from '@dxos/log';
 import { trace } from '@dxos/protocols';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { IdentityRecord, SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
-import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { AdmittedFeed, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
+import { trace as Trace } from '@dxos/tracing';
 import { deferFunction } from '@dxos/util';
 
 import { createAuthProvider } from './authenticator';
@@ -46,6 +48,7 @@ export type CreateIdentityOptions = {
 };
 
 // TODO(dmaretskyi): Rename: represents the peer's state machine.
+@Trace.resource()
 export class IdentityManager {
   readonly stateUpdate = new Event();
 
@@ -64,7 +67,8 @@ export class IdentityManager {
     return this._identity;
   }
 
-  async open() {
+  @Trace.span()
+  async open(ctx: Context) {
     const traceId = PublicKey.random().toHex();
     log.trace('dxos.halo.identity-manager.open', trace.begin({ id: traceId }));
     await this._metadataStore.load();
@@ -73,7 +77,7 @@ export class IdentityManager {
     log('identity record', { identityRecord });
     if (identityRecord) {
       this._identity = await this._constructIdentity(identityRecord);
-      await this._identity.open();
+      await this._identity.open(ctx);
       await this._identity.ready();
       log.trace('dxos.halo.identity', {
         identityKey: identityRecord.identityKey,
@@ -85,7 +89,7 @@ export class IdentityManager {
   }
 
   async close() {
-    await this._identity?.close();
+    await this._identity?.close(new Context());
   }
 
   async createIdentity({ displayName }: CreateIdentityOptions = {}) {
@@ -105,7 +109,7 @@ export class IdentityManager {
     };
 
     const identity = await this._constructIdentity(identityRecord);
-    await identity.open();
+    await identity.open(new Context());
 
     {
       const generator = new CredentialGenerator(this._keyring, identityRecord.identityKey, identityRecord.deviceKey);
@@ -175,7 +179,7 @@ export class IdentityManager {
     };
     const identity = await this._constructIdentity(identityRecord);
 
-    await identity.open();
+    await identity.open(new Context());
     this._identity = identity;
     await this._metadataStore.setIdentityRecord(identityRecord);
     await this._identity.ready();
@@ -189,6 +193,23 @@ export class IdentityManager {
     return identity;
   }
 
+  /**
+   * Update the profile document of an existing identity.
+   */
+  async updateProfile(profile: ProfileDocument) {
+    invariant(this._identity, 'Identity not initialized.');
+    // TODO(wittjosiah): Use CredentialGenerator.
+    const credential = await this._identity.getIdentityCredentialSigner().createCredential({
+      subject: this._identity.identityKey,
+      assertion: {
+        '@type': 'dxos.halo.credentials.IdentityProfile',
+        profile,
+      },
+    });
+    const receipt = await this._identity.controlPipeline.writer.write({ credential: { credential } });
+    await this._identity.controlPipeline.state.waitUntilTimeframe(new Timeframe([[receipt.feedKey, receipt.seq]]));
+  }
+
   private async _constructIdentity(identityRecord: IdentityRecord) {
     invariant(!this._identity);
     log('constructing identity', { identityRecord });

package/src/packlets/identity/identity-service.test.ts

@@ -6,6 +6,7 @@ import chai, { expect } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { Identity, IdentityService } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
@@ -22,8 +23,12 @@ describe('IdentityService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
-    identityService = new IdentityServiceImpl(serviceContext);
+    await serviceContext.open(new Context());
+    identityService = new IdentityServiceImpl(
+      (options) => serviceContext.createIdentity(options),
+      serviceContext.identityManager,
+      serviceContext.keyring,
+    );
   });
 
   afterEach(async () => {
@@ -54,6 +59,16 @@ describe('IdentityService', () => {
 
   describe.skip('recoverIdentity', () => {});
 
+  describe('updateProfile', () => {
+    test('updates profile', async () => {
+      const identity = await identityService.createIdentity({});
+      expect(identity.profile?.displayName).to.be.undefined;
+
+      const updatedIdentity = await identityService.updateProfile({ displayName: 'Example' });
+      expect(updatedIdentity.profile?.displayName).to.equal('Example');
+    });
+  });
+
   describe('queryIdentity', () => {
     test('returns undefined if no identity is available', async () => {
       const query = identityService.queryIdentity();

package/src/packlets/identity/identity-service.ts

@@ -6,6 +6,7 @@ import { Stream } from '@dxos/codec-protobuf';
 import { signPresentation } from '@dxos/credentials';
 import { todo } from '@dxos/debug';
 import { invariant } from '@dxos/invariant';
+import { Keyring } from '@dxos/keyring';
 import {
   Identity,
   IdentityService,
@@ -15,15 +16,17 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 import { Presentation, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 
-import { ServiceContext } from '../services';
+import { CreateIdentityOptions, IdentityManager } from './identity-manager';
 
 export class IdentityServiceImpl implements IdentityService {
-  // TODO(wittjosiah): Remove dependency on service context.
-  constructor(private readonly _serviceContext: ServiceContext) {}
+  constructor(
+    private readonly _createIdentity: (params?: CreateIdentityOptions) => Promise<Identity>,
+    private readonly _identityManager: IdentityManager,
+    private readonly _keyring: Keyring,
+  ) {}
 
   async createIdentity(request: ProfileDocument): Promise<Identity> {
-    await this._serviceContext.createIdentity(request);
-
+    await this._createIdentity(request);
     return this._getIdentity()!;
   }
 
@@ -36,30 +39,36 @@ export class IdentityServiceImpl implements IdentityService {
       const emitNext = () => next({ identity: this._getIdentity() });
 
       emitNext();
-      return this._serviceContext.identityManager.stateUpdate.on(emitNext);
+      return this._identityManager.stateUpdate.on(emitNext);
     });
   }
 
   private _getIdentity(): Identity | undefined {
-    if (!this._serviceContext.identityManager.identity) {
+    if (!this._identityManager.identity) {
       return undefined;
     }
 
     return {
-      identityKey: this._serviceContext.identityManager.identity.identityKey,
-      spaceKey: this._serviceContext.identityManager.identity.space.key,
-      profile: this._serviceContext.identityManager.identity.profileDocument,
+      identityKey: this._identityManager.identity.identityKey,
+      spaceKey: this._identityManager.identity.space.key,
+      profile: this._identityManager.identity.profileDocument,
     };
   }
 
+  async updateProfile(profile: ProfileDocument): Promise<Identity> {
+    invariant(this._identityManager.identity, 'Identity not initialized.');
+    await this._identityManager.updateProfile(profile);
+    return this._getIdentity()!;
+  }
+
   async signPresentation({ presentation, nonce }: SignPresentationRequest): Promise<Presentation> {
-    invariant(this._serviceContext.identityManager.identity, 'Identity not initialized.');
+    invariant(this._identityManager.identity, 'Identity not initialized.');
 
     return await signPresentation({
       presentation,
-      signer: this._serviceContext.keyring,
-      signerKey: this._serviceContext.identityManager.identity.deviceKey,
-      chain: this._serviceContext.identityManager.identity.deviceCredentialChain,
+      signer: this._keyring,
+      signerKey: this._identityManager.identity.deviceKey,
+      chain: this._identityManager.identity.deviceCredentialChain,
       nonce,
     });
   }

package/src/packlets/identity/identity.test.ts

@@ -4,6 +4,7 @@
 
 import expect from 'expect';
 
+import { Context } from '@dxos/context';
 import { CredentialGenerator, verifyCredential } from '@dxos/credentials';
 import {
   MetadataStore,
@@ -106,8 +107,8 @@ describe('identity/identity', () => {
       space,
     });
 
-    await identity.open();
-    afterTest(() => identity.close());
+    await identity.open(new Context());
+    afterTest(() => identity.close(new Context()));
 
     //
     // Identity genesis
@@ -219,8 +220,8 @@ describe('identity/identity', () => {
         space,
       }));
 
-      await identity.open();
-      afterTest(() => identity.close());
+      await identity.open(new Context());
+      afterTest(() => identity.close(new Context()));
 
       //
       // Identity genesis
@@ -310,8 +311,8 @@ describe('identity/identity', () => {
         space,
       }));
 
-      await identity.open();
-      afterTest(() => identity.close());
+      await identity.open(new Context());
+      afterTest(() => identity.close(new Context()));
     }
 
     //

package/src/packlets/identity/identity.ts

@@ -4,6 +4,7 @@
 
 import { Event } from '@dxos/async';
 import { AUTH_TIMEOUT, LOAD_CONTROL_FEEDS_TIMEOUT } from '@dxos/client-protocol';
+import { Context } from '@dxos/context';
 import {
   DeviceStateMachine,
   CredentialSigner,
@@ -21,6 +22,7 @@ import { log } from '@dxos/log';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { DeviceAdmissionRequest } from '@dxos/protocols/proto/dxos/halo/invitations';
+import { trace } from '@dxos/tracing';
 import { ComplexSet } from '@dxos/util';
 
 import { TrustedKeySetAuthVerifier } from './authenticator';
@@ -35,6 +37,7 @@ export type IdentityParams = {
 /**
  * Agent identity manager, which includes the agent's Halo space.
  */
+@trace.resource()
 export class Identity {
   public readonly space: Space;
   private readonly _signer: Signer;
@@ -76,13 +79,15 @@ export class Identity {
     return this._deviceStateMachine.authorizedDeviceKeys;
   }
 
-  async open() {
+  @trace.span()
+  async open(ctx: Context) {
     await this.space.spaceState.addCredentialProcessor(this._deviceStateMachine);
     await this.space.spaceState.addCredentialProcessor(this._profileStateMachine);
-    await this.space.open();
+    await this.space.open(ctx);
   }
 
-  async close() {
+  @trace.span()
+  async close(ctx: Context) {
     await this.authVerifier.close();
     await this.space.spaceState.removeCredentialProcessor(this._profileStateMachine);
     await this.space.spaceState.removeCredentialProcessor(this._deviceStateMachine);

package/src/packlets/invitations/device-invitation-protocol.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { asyncChain } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test, afterTest } from '@dxos/test';
 
@@ -19,7 +20,7 @@ const closeAfterTest = async (peer: ServiceContext) => {
 describe('services/device', () => {
   test('creates identity', async () => {
     const peer = createServiceContext();
-    await peer.open();
+    await peer.open(new Context());
     afterTest(() => peer.close());
 
     const identity = await peer.createIdentity();

package/src/packlets/logging/logging-service.ts

@@ -13,6 +13,7 @@ import {
   ControlMetricsRequest,
   ControlMetricsResponse,
   QueryMetricsRequest,
+  QueryMetricsResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
 import { jsonify, numericalValues, tracer } from '@dxos/util';
 
@@ -21,6 +22,7 @@ import { jsonify, numericalValues, tracer } from '@dxos/util';
  */
 export class LoggingServiceImpl implements LoggingService {
   private readonly _logs = new Event<NaturalLogEntry>();
+  private readonly _started = new Date();
 
   async open() {
     log.runtimeConfig.processors.push(this._logProcessor);
@@ -45,11 +47,14 @@ export class LoggingServiceImpl implements LoggingService {
     return { recording: tracer.recording };
   }
 
-  queryMetrics({ interval }: QueryMetricsRequest): Stream<Metrics> {
+  /**
+   * @deprecated (Move to diagnostics).
+   */
+  queryMetrics({ interval = 5_000 }: QueryMetricsRequest): Stream<QueryMetricsResponse> {
     // TODO(burdon): Map all traces; how to bind to reducer/metrics shape (e.g., numericalValues)?
-    const createNumericalValues = (key: string) => {
-      const consume = tracer.get(key) ?? [];
-      return { key, stats: numericalValues(consume, 'duration') };
+    const getNumericalValues = (key: string) => {
+      const events = tracer.get(key) ?? [];
+      return { key, stats: numericalValues(events, 'duration') };
     };
 
     return new Stream(({ next }) => {
@@ -57,16 +62,19 @@ export class LoggingServiceImpl implements LoggingService {
         const metrics: Metrics = {
           timestamp: new Date(),
           values: [
-            createNumericalValues('dxos.echo.pipeline.control'),
-            createNumericalValues('dxos.echo.pipeline.data'),
-          ],
+            getNumericalValues('dxos.echo.pipeline.control'),
+            getNumericalValues('dxos.echo.pipeline.data'),
+          ].filter(Boolean) as Metrics.KeyPair[],
         };
 
-        next(metrics);
+        next({
+          timestamp: new Date(),
+          metrics,
+        });
       };
 
       update();
-      const i = setInterval(update, interval);
+      const i = setInterval(update, Math.max(interval, 1_000));
       return () => {
         clearInterval(i);
       };

package/src/packlets/network/network-service.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { NetworkService, ConnectionState } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
 
@@ -18,7 +19,7 @@ describe('NetworkService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     networkService = new NetworkServiceImpl(serviceContext.networkManager, serviceContext.signalManager);
   });
 

package/src/packlets/services/client-rpc-server.ts

@@ -5,7 +5,7 @@
 import { ClientServices } from '@dxos/client-protocol';
 import { Any, ServiceHandler, Stream } from '@dxos/codec-protobuf';
 import { raise } from '@dxos/debug';
-import { parseMethodName, RpcPeer, RpcPeerOptions } from '@dxos/rpc';
+import { parseMethodName, RpcPeer, RpcPeerOptions, ServiceBundle } from '@dxos/rpc';
 import { MaybePromise } from '@dxos/util';
 
 import { ServiceRegistry } from './service-registry';
@@ -74,16 +74,16 @@ export class ClientRpcServer {
   private _getServiceHandler(serviceName: string) {
     if (!this._handlerCache.has(serviceName)) {
       const [key, descriptor] =
-        Object.entries(this._serviceRegistry.descriptors).find(
+        Object.entries(this._serviceRegistry.descriptors as ServiceBundle<Record<string, any>>).find(
           ([key, descriptor]) => descriptor.name === serviceName,
         ) ?? raise(new Error(`Service not available: ${serviceName}`));
 
-      const service = this._serviceRegistry.services[key as keyof ClientServices];
+      const service = this._serviceRegistry.services[key as keyof ClientServices] as any;
       if (!service) {
         throw new Error(`Service not available: ${serviceName}`);
       }
 
-      this._handlerCache.set(serviceName, descriptor.createServer(service as any));
+      this._handlerCache.set(serviceName, descriptor.createServer(service));
     }
 
     return this._handlerCache.get(serviceName)!;

package/src/packlets/services/diagnostics.ts

@@ -0,0 +1,211 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { ClientServices } from '@dxos/client-protocol';
+import { getFirstStreamValue } from '@dxos/codec-protobuf';
+import { Config, ConfigProto } from '@dxos/config';
+import { credentialTypeFilter } from '@dxos/credentials';
+import { DocumentModel, DocumentModelState } from '@dxos/document-model';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { STORAGE_VERSION } from '@dxos/protocols';
+import {
+  Device,
+  Identity,
+  Metrics,
+  Space as SpaceProto,
+  SpaceMember,
+} from '@dxos/protocols/proto/dxos/client/services';
+import { SubscribeToFeedsResponse } from '@dxos/protocols/proto/dxos/devtools/host';
+import { Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
+
+import { DXOS_VERSION } from '../../version';
+import { DataSpace } from '../spaces';
+import { getPlatform, Platform } from './platform';
+import { ServiceContext } from './service-context';
+
+const DEFAULT_TIMEOUT = 1_000;
+
+export type Diagnostics = {
+  created: string;
+  platform: Platform;
+  config?: ConfigProto;
+  client: {
+    version: string;
+    storage: {
+      version: number;
+    };
+  };
+  identity?: Identity;
+  devices?: Device[];
+  spaces?: SpaceStats[];
+  feeds?: Partial<SubscribeToFeedsResponse.Feed>[];
+  metrics?: Metrics;
+};
+
+// TODO(burdon): Normalize for ECHO/HALO.
+export type SpaceStats = {
+  key: PublicKey;
+  properties?: {
+    name: string;
+  };
+  db?: {
+    objects: number;
+  };
+  metrics?: SpaceProto.Metrics & {
+    startupTime?: number;
+  };
+  epochs?: (Epoch & { id?: PublicKey })[];
+  members?: SpaceMember[];
+  pipeline?: SpaceProto.PipelineState;
+};
+
+/**
+ * Create diagnostics to provide snapshot of current system state.
+ */
+export const createDiagnostics = async (
+  clientServices: Partial<ClientServices>,
+  serviceContext: ServiceContext,
+  config: Config,
+): Promise<Diagnostics> => {
+  const diagnostics: Diagnostics = {
+    created: new Date().toISOString(),
+    platform: getPlatform(),
+    client: {
+      version: DXOS_VERSION,
+      storage: {
+        version: STORAGE_VERSION,
+      },
+    },
+  };
+
+  // Trace metrics.
+  // TODO(burdon): Move here from logging service?
+  {
+    invariant(clientServices.LoggingService, 'SystemService is not available.');
+    diagnostics.metrics = await getFirstStreamValue(clientServices.LoggingService.queryMetrics({}), {
+      timeout: DEFAULT_TIMEOUT,
+    }).catch(() => undefined);
+  }
+
+  const identity = serviceContext.identityManager.identity;
+  if (identity) {
+    // Identity.
+    diagnostics.identity = {
+      identityKey: identity.identityKey,
+      spaceKey: identity.space.key,
+      profile: identity.profileDocument,
+    };
+
+    // Devices.
+    const { devices } =
+      (await getFirstStreamValue(clientServices.DevicesService!.queryDevices(), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.devices = devices;
+
+    // TODO(dmaretskyi): Add metrics for halo space.
+
+    // Spaces.
+    if (serviceContext.dataSpaceManager) {
+      diagnostics.spaces = await Promise.all(
+        Array.from(serviceContext.dataSpaceManager.spaces.values()).map((space) => getSpaceStats(space)) ?? [],
+      );
+    }
+
+    // Feeds.
+    const { feeds = [] } =
+      (await getFirstStreamValue(clientServices.DevtoolsHost!.subscribeToFeeds({}), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.feeds = feeds.map(({ feedKey, bytes, length }) => ({ feedKey, bytes, length }));
+  }
+
+  diagnostics.config = config.values;
+
+  return diagnostics;
+};
+
+const getProperties = (space: DataSpace) => {
+  let properties: any = {};
+  try {
+    // Add properties to cache.
+    const propertiesItem = space.dataPipeline.itemManager.items.find(
+      (item) =>
+        item.modelMeta?.type === DocumentModel.meta.type &&
+        (item.state as DocumentModelState)?.type === 'dxos.sdk.client.Properties',
+    );
+
+    const state = propertiesItem?.state as DocumentModelState;
+    properties = state?.data;
+  } catch (err: any) {
+    log.warn(err.message);
+  }
+
+  return properties;
+};
+
+const getSpaceStats = async (space: DataSpace): Promise<SpaceStats> => {
+  const stats: SpaceStats = {
+    key: space.key,
+    metrics: space.metrics,
+
+    epochs: space.inner.spaceState.credentials
+      .filter(credentialTypeFilter('dxos.halo.credentials.Epoch'))
+      .map((credential) => ({
+        ...credential.subject.assertion,
+        id: credential.id,
+      })),
+
+    members: Array.from(space.inner.spaceState.members.values()).map((member) => ({
+      identity: {
+        identityKey: member.key,
+        profile: {
+          displayName: member.assertion.profile?.displayName,
+        },
+      },
+      presence:
+        space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key)).length > 0
+          ? SpaceMember.PresenceState.ONLINE
+          : SpaceMember.PresenceState.OFFLINE,
+    })),
+
+    pipeline: {
+      // TODO(burdon): Pick properties from credentials if needed.
+      // currentEpoch: space.dataPipeline.currentEpoch,
+      // appliedEpoch: space.dataPipeline.appliedEpoch,
+
+      controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
+      currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
+      targetControlTimeframe: space.inner.controlPipeline.state.targetTimeframe,
+      totalControlTimeframe: space.inner.controlPipeline.state.endTimeframe,
+
+      // TODO(burdon): Empty?
+      dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
+      startDataTimeframe: space.dataPipeline.pipelineState?.startTimeframe,
+      currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
+      targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
+      totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
+    },
+  };
+
+  // TODO(burdon): May not be open?
+  if (space.dataPipeline.itemManager) {
+    Object.assign(stats, {
+      properties: getProperties(space),
+      db: {
+        objects: space.dataPipeline.itemManager.entities.size,
+      },
+    } as SpaceStats);
+  }
+
+  // TODO(burdon): Factor out.
+  if (stats.metrics) {
+    const { open, ready } = stats.metrics;
+    stats.metrics.startupTime = open && ready && ready.getTime() - open.getTime();
+  }
+
+  return stats;
+};

package/src/packlets/services/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './client-rpc-server';
+export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';