@dxos/client-services 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a

package/src/packlets/network/network-service.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { NetworkService, ConnectionState } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
 
@@ -18,7 +19,7 @@ describe('NetworkService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     networkService = new NetworkServiceImpl(serviceContext.networkManager, serviceContext.signalManager);
   });
 

package/src/packlets/services/client-rpc-server.ts

@@ -5,7 +5,7 @@
 import { ClientServices } from '@dxos/client-protocol';
 import { Any, ServiceHandler, Stream } from '@dxos/codec-protobuf';
 import { raise } from '@dxos/debug';
-import { parseMethodName, RpcPeer, RpcPeerOptions } from '@dxos/rpc';
+import { parseMethodName, RpcPeer, RpcPeerOptions, ServiceBundle } from '@dxos/rpc';
 import { MaybePromise } from '@dxos/util';
 
 import { ServiceRegistry } from './service-registry';
@@ -74,16 +74,16 @@ export class ClientRpcServer {
   private _getServiceHandler(serviceName: string) {
     if (!this._handlerCache.has(serviceName)) {
       const [key, descriptor] =
-        Object.entries(this._serviceRegistry.descriptors).find(
+        Object.entries(this._serviceRegistry.descriptors as ServiceBundle<Record<string, any>>).find(
           ([key, descriptor]) => descriptor.name === serviceName,
         ) ?? raise(new Error(`Service not available: ${serviceName}`));
 
-      const service = this._serviceRegistry.services[key as keyof ClientServices];
+      const service = this._serviceRegistry.services[key as keyof ClientServices] as any;
       if (!service) {
         throw new Error(`Service not available: ${serviceName}`);
       }
 
-      this._handlerCache.set(serviceName, descriptor.createServer(service as any));
+      this._handlerCache.set(serviceName, descriptor.createServer(service));
     }
 
     return this._handlerCache.get(serviceName)!;

package/src/packlets/services/diagnostics.ts

@@ -0,0 +1,212 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { ClientServices } from '@dxos/client-protocol';
+import { getFirstStreamValue } from '@dxos/codec-protobuf';
+import { Config, ConfigProto } from '@dxos/config';
+import { credentialTypeFilter } from '@dxos/credentials';
+import { DocumentModel, DocumentModelState } from '@dxos/document-model';
+import { TYPE_PROPERTIES } from '@dxos/echo-db';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { STORAGE_VERSION } from '@dxos/protocols';
+import {
+  Device,
+  Identity,
+  Metrics,
+  Space as SpaceProto,
+  SpaceMember,
+} from '@dxos/protocols/proto/dxos/client/services';
+import { SubscribeToFeedsResponse } from '@dxos/protocols/proto/dxos/devtools/host';
+import { Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
+
+import { DXOS_VERSION } from '../../version';
+import { DataSpace } from '../spaces';
+import { getPlatform, Platform } from './platform';
+import { ServiceContext } from './service-context';
+
+const DEFAULT_TIMEOUT = 1_000;
+
+export type Diagnostics = {
+  created: string;
+  platform: Platform;
+  config?: ConfigProto;
+  client: {
+    version: string;
+    storage: {
+      version: number;
+    };
+  };
+  identity?: Identity;
+  devices?: Device[];
+  spaces?: SpaceStats[];
+  feeds?: Partial<SubscribeToFeedsResponse.Feed>[];
+  metrics?: Metrics;
+};
+
+// TODO(burdon): Normalize for ECHO/HALO.
+export type SpaceStats = {
+  key: PublicKey;
+  properties?: {
+    name: string;
+  };
+  db?: {
+    objects: number;
+  };
+  metrics?: SpaceProto.Metrics & {
+    startupTime?: number;
+  };
+  epochs?: (Epoch & { id?: PublicKey })[];
+  members?: SpaceMember[];
+  pipeline?: SpaceProto.PipelineState;
+};
+
+/**
+ * Create diagnostics to provide snapshot of current system state.
+ */
+export const createDiagnostics = async (
+  clientServices: Partial<ClientServices>,
+  serviceContext: ServiceContext,
+  config: Config,
+): Promise<Diagnostics> => {
+  const diagnostics: Diagnostics = {
+    created: new Date().toISOString(),
+    platform: getPlatform(),
+    client: {
+      version: DXOS_VERSION,
+      storage: {
+        version: STORAGE_VERSION,
+      },
+    },
+  };
+
+  // Trace metrics.
+  // TODO(burdon): Move here from logging service?
+  {
+    invariant(clientServices.LoggingService, 'SystemService is not available.');
+    diagnostics.metrics = await getFirstStreamValue(clientServices.LoggingService.queryMetrics({}), {
+      timeout: DEFAULT_TIMEOUT,
+    }).catch(() => undefined);
+  }
+
+  const identity = serviceContext.identityManager.identity;
+  if (identity) {
+    // Identity.
+    diagnostics.identity = {
+      identityKey: identity.identityKey,
+      spaceKey: identity.space.key,
+      profile: identity.profileDocument,
+    };
+
+    // Devices.
+    const { devices } =
+      (await getFirstStreamValue(clientServices.DevicesService!.queryDevices(), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.devices = devices;
+
+    // TODO(dmaretskyi): Add metrics for halo space.
+
+    // Spaces.
+    if (serviceContext.dataSpaceManager) {
+      diagnostics.spaces = await Promise.all(
+        Array.from(serviceContext.dataSpaceManager.spaces.values()).map((space) => getSpaceStats(space)) ?? [],
+      );
+    }
+
+    // Feeds.
+    const { feeds = [] } =
+      (await getFirstStreamValue(clientServices.DevtoolsHost!.subscribeToFeeds({}), {
+        timeout: DEFAULT_TIMEOUT,
+      }).catch(() => undefined)) ?? {};
+    diagnostics.feeds = feeds.map(({ feedKey, bytes, length }) => ({ feedKey, bytes, length }));
+  }
+
+  diagnostics.config = config.values;
+
+  return diagnostics;
+};
+
+const getProperties = (space: DataSpace) => {
+  let properties: any = {};
+  try {
+    // Add properties to cache.
+    const propertiesItem = space.dataPipeline.itemManager.items.find(
+      (item) =>
+        item.modelMeta?.type === DocumentModel.meta.type &&
+        (item.state as DocumentModelState)?.type === TYPE_PROPERTIES,
+    );
+
+    const state = propertiesItem?.state as DocumentModelState;
+    properties = state?.data;
+  } catch (err: any) {
+    log.warn(err.message);
+  }
+
+  return properties;
+};
+
+const getSpaceStats = async (space: DataSpace): Promise<SpaceStats> => {
+  const stats: SpaceStats = {
+    key: space.key,
+    metrics: space.metrics,
+
+    epochs: space.inner.spaceState.credentials
+      .filter(credentialTypeFilter('dxos.halo.credentials.Epoch'))
+      .map((credential) => ({
+        ...credential.subject.assertion,
+        id: credential.id,
+      })),
+
+    members: Array.from(space.inner.spaceState.members.values()).map((member) => ({
+      identity: {
+        identityKey: member.key,
+        profile: {
+          displayName: member.assertion.profile?.displayName,
+        },
+      },
+      presence:
+        space.presence.getPeersOnline().filter(({ identityKey }) => identityKey.equals(member.key)).length > 0
+          ? SpaceMember.PresenceState.ONLINE
+          : SpaceMember.PresenceState.OFFLINE,
+    })),
+
+    pipeline: {
+      // TODO(burdon): Pick properties from credentials if needed.
+      // currentEpoch: space.dataPipeline.currentEpoch,
+      // appliedEpoch: space.dataPipeline.appliedEpoch,
+
+      controlFeeds: space.inner.controlPipeline.state.feeds.map((feed) => feed.key),
+      currentControlTimeframe: space.inner.controlPipeline.state.timeframe,
+      targetControlTimeframe: space.inner.controlPipeline.state.targetTimeframe,
+      totalControlTimeframe: space.inner.controlPipeline.state.endTimeframe,
+
+      // TODO(burdon): Empty?
+      dataFeeds: space.dataPipeline.pipelineState?.feeds.map((feed) => feed.key) ?? [],
+      startDataTimeframe: space.dataPipeline.pipelineState?.startTimeframe,
+      currentDataTimeframe: space.dataPipeline.pipelineState?.timeframe,
+      targetDataTimeframe: space.dataPipeline.pipelineState?.targetTimeframe,
+      totalDataTimeframe: space.dataPipeline.pipelineState?.endTimeframe,
+    },
+  };
+
+  // TODO(burdon): May not be open?
+  if (space.dataPipeline.itemManager) {
+    Object.assign(stats, {
+      properties: getProperties(space),
+      db: {
+        objects: space.dataPipeline.itemManager.entities.size,
+      },
+    } as SpaceStats);
+  }
+
+  // TODO(burdon): Factor out.
+  if (stats.metrics) {
+    const { open, ready } = stats.metrics;
+    stats.metrics.startupTime = open && ready && ready.getTime() - open.getTime();
+  }
+
+  return stats;
+};

package/src/packlets/services/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './client-rpc-server';
+export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';

package/src/packlets/services/platform.ts

@@ -0,0 +1,48 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export type Platform = {
+  type: 'browser' | 'shared-worker' | 'node';
+  userAgent?: string;
+  platform?: string;
+  runtime?: string;
+  uptime?: number;
+  memory?: {
+    rss: number;
+    heapTotal: number;
+    heapUsed: number;
+    external: number;
+    arrayBuffers: number;
+  };
+};
+
+export const getPlatform = (): Platform => {
+  if ((process as any).browser) {
+    if (typeof window !== 'undefined') {
+      // Browser.
+      const { userAgent } = window.navigator;
+      return {
+        type: 'browser',
+        userAgent,
+        uptime: Math.floor((Date.now() - window.performance.timeOrigin) / 1_000),
+      };
+    } else {
+      // Shared worker.
+      return {
+        type: 'shared-worker',
+        uptime: Math.floor((Date.now() - performance.timeOrigin) / 1_000),
+      };
+    }
+  } else {
+    // Node.
+    const { platform, version, arch } = process;
+    return {
+      type: 'node',
+      platform: `${platform} ${version} ${arch}`,
+      runtime: process.version,
+      uptime: Math.floor(process.uptime()),
+      memory: process.memoryUsage(),
+    };
+  }
+};

package/src/packlets/services/service-context.test.ts

@@ -2,9 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
+import { expect } from 'chai';
+
+import { DocumentModel, DocumentModelState, MutationBuilder } from '@dxos/document-model';
+import { createModelMutation, encodeModelMutation, genesisMutation } from '@dxos/echo-db';
+import { WriteReceipt } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
 
 import { createServiceContext, syncItemsLocal } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -16,11 +25,61 @@ describe('services/ServiceContext', () => {
     await device1.createIdentity();
     const space1 = await device1.dataSpaceManager!.createSpace();
 
+    const itemId = PublicKey.random().toHex();
+    await space1.dataPipeline.databaseHost!.getWriteStream()?.write({
+      batch: genesisMutation(itemId, DocumentModel.meta.type),
+    });
+
+    let counter = 0;
+
+    for (const _ in range(5)) {
+      const receipts: WriteReceipt[] = [];
+      for (const _ in range(50)) {
+        receipts.push(
+          await space1.dataPipeline.databaseHost!.getWriteStream()!.write({
+            batch: createModelMutation(
+              itemId,
+              encodeModelMutation(DocumentModel.meta, new MutationBuilder().set('counter', ++counter).build()),
+            ),
+          }),
+        );
+      }
+
+      await space1.dataPipeline.pipelineState!.waitUntilTimeframe(
+        Timeframe.merge(...receipts.map((receipt) => new Timeframe([[receipt.feedKey, receipt.seq]]))),
+      );
+
+      await space1.createEpoch();
+      log.info('epoch', { number: space1.dataPipeline.currentEpoch?.subject.assertion.number });
+    }
+
     const device2 = createServiceContext({ signalContext: networkContext });
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     await device2.dataSpaceManager!.waitUntilSpaceReady(space1!.key);
     const space2 = await device2.dataSpaceManager!.spaces.get(space1.key);
+
+    log.info('peer 2', {
+      currentEpoch: space2!.dataPipeline.currentEpoch?.subject.assertion.number,
+      appliedEpoch: space2!.dataPipeline.appliedEpoch?.subject.assertion.number,
+    });
+
+    await space2?.dataPipeline.onNewEpoch.waitForCondition(
+      () =>
+        space2!.dataPipeline.appliedEpoch?.subject.assertion.number ===
+        space1.dataPipeline.currentEpoch?.subject.assertion.number,
+    );
+
+    expect(space2!.dataPipeline.currentEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.currentEpoch!.subject.assertion.number,
+    );
+    expect(space2!.dataPipeline.appliedEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.appliedEpoch!.subject.assertion.number,
+    );
+
+    const item2 = space2!.dataPipeline.itemManager.entities.get(itemId);
+    expect((item2!.state as DocumentModelState).data.counter).to.equal(counter);
+
     await syncItemsLocal(space1.dataPipeline, space2!.dataPipeline);
   });
 

package/src/packlets/services/service-context.ts

@@ -2,9 +2,8 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { CredentialProcessor, getCredentialAssertion } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import {
@@ -15,6 +14,7 @@ import {
   SnapshotStore,
 } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -27,6 +27,8 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Storage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { trace as Trace } from '@dxos/tracing';
+import { safeInstanceof } from '@dxos/util';
 
 import { CreateIdentityOptions, IdentityManager, JoinIdentityParams } from '../identity';
 import {
@@ -41,6 +43,9 @@ import { DataSpaceManager, SigningContext } from '../spaces';
  * Shared backend for all client services.
  */
 // TODO(burdon): Rename/break-up into smaller components. And/or make members private.
+// TODO(dmaretskyi): Gets duplicated in CJS build between normal and testing bundles.
+@safeInstanceof('dxos.client-services.ServiceContext')
+@Trace.resource()
 export class ServiceContext {
   public readonly initialized = new Trigger();
   public readonly dataServiceSubscriptions = new DataServiceSubscriptions();
@@ -86,8 +91,9 @@ export class ServiceContext {
         root: storage.createDirectory('feeds'),
         signer: this.keyring,
         hypercore: {
-          valueEncoding
-        }
+          valueEncoding,
+          stats: true,
+        },
       })
     });
 
@@ -122,21 +128,21 @@ export class ServiceContext {
     );
   }
 
-  async open() {
-    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
-
+  @Trace.span()
+  async open(ctx: Context) {
     await this._checkStorageVersion();
 
     log('opening...');
+    log.trace('dxos.sdk.service-context.open', trace.begin({ id: this._instanceId }));
     await this.signalManager.open();
     await this.networkManager.open();
     await this.spaceManager.open();
-    await this.identityManager.open();
+    await this.identityManager.open(ctx);
     if (this.identityManager.identity) {
-      await this._initialize();
+      await this._initialize(ctx);
     }
-    log('opened');
     log.trace('dxos.sdk.service-context.open', trace.end({ id: this._instanceId }));
+    log('opened');
   }
 
   async close() {
@@ -156,8 +162,7 @@ export class ServiceContext {
 
   async createIdentity(params: CreateIdentityOptions = {}) {
     const identity = await this.identityManager.createIdentity(params);
-
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -169,8 +174,7 @@ export class ServiceContext {
 
   private async _acceptIdentity(params: JoinIdentityParams) {
     const identity = await this.identityManager.acceptIdentity(params);
-
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -183,7 +187,8 @@ export class ServiceContext {
   }
 
   // Called when identity is created.
-  private async _initialize() {
+  @Trace.span()
+  private async _initialize(ctx: Context) {
     log('initializing spaces...');
     const identity = this.identityManager.identity ?? failUndefined();
     const signingContext: SigningContext = {
@@ -242,6 +247,7 @@ export class ServiceContext {
         }
       },
     };
+
     await identity.space.spaceState.addCredentialProcessor(this._deviceSpaceSync);
   }
 }

package/src/packlets/services/service-host.test.ts

@@ -8,6 +8,7 @@ import { rmSync } from 'node:fs';
 
 import { asyncTimeout, latch, Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { verifyPresentation } from '@dxos/credentials';
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManagerContext } from '@dxos/messaging';
@@ -30,7 +31,7 @@ describe('ClientServicesHost', () => {
 
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
@@ -49,14 +50,14 @@ describe('ClientServicesHost', () => {
 
   test('write and query credentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     // Test if Identity exposes haloSpace key.
@@ -86,14 +87,14 @@ describe('ClientServicesHost', () => {
 
   test('sign presentation', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     const nonce = new Uint8Array([0, 0, 0, 0]);
@@ -117,11 +118,11 @@ describe('ClientServicesHost', () => {
     });
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
 
       await host.services.IdentityService?.createIdentity({});
 
-      expect(host._serviceContext.storage.size).to.exist;
+      expect(host.context.storage.size).to.exist;
 
       await asyncTimeout(host.reset(), 1000);
       await host.close();
@@ -129,7 +130,7 @@ describe('ClientServicesHost', () => {
 
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
       const trigger = new Trigger<Identity>();
 
       const stream = host.services.IdentityService?.queryIdentity();
@@ -141,7 +142,7 @@ describe('ClientServicesHost', () => {
         }
       });
       await expect(asyncTimeout(trigger.wait(), 200)).to.be.rejectedWith();
-      stream?.close();
+      await stream?.close();
       await host.close();
     }
   }).onlyEnvironments('nodejs', 'chromium', 'firefox');