@dxos/client-services 0.1.56-main.bf228a7 → 0.1.56-main.c19b7cd

package/src/packlets/services/service-host.test.ts

@@ -8,6 +8,7 @@ import { rmSync } from 'node:fs';
 
 import { asyncTimeout, latch, Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { verifyPresentation } from '@dxos/credentials';
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManagerContext } from '@dxos/messaging';
@@ -30,7 +31,7 @@ describe('ClientServicesHost', () => {
 
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
@@ -49,14 +50,14 @@ describe('ClientServicesHost', () => {
 
   test('write and query credentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     // Test if Identity exposes haloSpace key.
@@ -86,14 +87,14 @@ describe('ClientServicesHost', () => {
 
   test('sign presentation', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
 
     const testCredential = await createMockCredential({
-      signer: host._serviceContext.keyring,
-      issuer: host._serviceContext.identityManager.identity!.deviceKey,
+      signer: host.context.keyring,
+      issuer: host.context.identityManager.identity!.deviceKey,
     });
 
     const nonce = new Uint8Array([0, 0, 0, 0]);
@@ -117,11 +118,11 @@ describe('ClientServicesHost', () => {
     });
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
 
       await host.services.IdentityService?.createIdentity({});
 
-      expect(host._serviceContext.storage.size).to.exist;
+      expect(host.context.storage.size).to.exist;
 
       await asyncTimeout(host.reset(), 1000);
       await host.close();
@@ -129,7 +130,7 @@ describe('ClientServicesHost', () => {
 
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
       const trigger = new Trigger<Identity>();
 
       const stream = host.services.IdentityService?.queryIdentity();

package/src/packlets/services/service-host.ts

@@ -5,6 +5,7 @@
 import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, ClientServices } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { DocumentModel } from '@dxos/document-model';
 import { DataServiceImpl } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
@@ -17,6 +18,7 @@ import { trace } from '@dxos/protocols';
 import { SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { Storage } from '@dxos/random-access-storage';
 import { TextModel } from '@dxos/text-model';
+import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
 
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsServiceImpl, DevtoolsHostEvents } from '../devtools';
@@ -28,6 +30,7 @@ import { NetworkServiceImpl } from '../network';
 import { SpacesServiceImpl } from '../spaces';
 import { createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
+import { createDiagnostics } from './diagnostics';
 import { ServiceContext } from './service-context';
 import { ServiceRegistry } from './service-registry';
 
@@ -64,11 +67,13 @@ export type InitializeOptions = {
 /**
  * Remote service implementation.
  */
+@Trace.resource()
 export class ClientServicesHost {
   private readonly _resourceLock?: ResourceLock;
   private readonly _serviceRegistry: ServiceRegistry<ClientServices>;
   private readonly _systemService: SystemServiceImpl;
   private readonly _loggingService: LoggingServiceImpl;
+  private readonly _tracingService = TRACE_PROCESSOR.createTraceSender();
 
   private _config?: Config;
   private readonly _statusUpdate = new Event<void>();
@@ -78,17 +83,19 @@ export class ClientServicesHost {
   private _storage?: Storage;
   private _callbacks?: ClientServicesHostCallbacks;
 
-  _serviceContext!: ServiceContext;
+  private _serviceContext!: ServiceContext;
+
+  @Trace.info()
   private _opening = false;
+
+  @Trace.info()
   private _open = false;
 
   constructor({
     config,
     modelFactory = createDefaultModelFactory(),
-    // TODO(burdon): Create ApolloLink abstraction (see Client).
     transportFactory,
     signalManager,
-    connectionLog,
     storage,
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
@@ -107,7 +114,7 @@ export class ClientServicesHost {
         lockKey,
         onAcquire: () => {
           if (!this._opening) {
-            void this.open();
+            void this.open(new Context());
           }
         },
         onRelease: () => this.close(),
@@ -116,11 +123,11 @@ export class ClientServicesHost {
 
     this._systemService = new SystemServiceImpl({
       config: this._config,
-
       statusUpdate: this._statusUpdate,
-
       getCurrentStatus: () => (this.isOpen ? SystemStatus.ACTIVE : SystemStatus.INACTIVE),
-
+      getDiagnostics: () => {
+        return createDiagnostics(this._serviceRegistry.services, this._serviceContext, this._config!);
+      },
       onUpdateStatus: async (status: SystemStatus) => {
         if (!this.isOpen && status === SystemStatus.ACTIVE) {
           await this._resourceLock?.acquire();
@@ -128,7 +135,6 @@ export class ClientServicesHost {
           await this._resourceLock?.release();
         }
       },
-
       onReset: async () => {
         await this.reset();
       },
@@ -136,9 +142,9 @@ export class ClientServicesHost {
 
     this._loggingService = new LoggingServiceImpl();
 
-    // TODO(burdon): Start to think of DMG (dynamic services).
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
       SystemService: this._systemService,
+      TracingService: this._tracingService,
     });
   }
 
@@ -146,6 +152,14 @@ export class ClientServicesHost {
     return this._open;
   }
 
+  get config() {
+    return this._config;
+  }
+
+  get context() {
+    return this._serviceContext;
+  }
+
   get serviceRegistry() {
     return this._serviceRegistry;
   }
@@ -165,10 +179,10 @@ export class ClientServicesHost {
    */
   initialize({ config, ...options }: InitializeOptions) {
     invariant(!this._open, 'service host is open');
+    log.info('initializing...');
 
     if (config) {
       invariant(!this._config, 'config already set');
-
       this._config = config;
       if (!this._storage) {
         this._storage = createStorageObjects(config.get('runtime.client.storage', {})!).storage;
@@ -190,16 +204,19 @@ export class ClientServicesHost {
       transportFactory,
       signalManager,
     });
+
+    log.info('initialized');
   }
 
   @synchronized
-  async open() {
+  @Trace.span()
+  async open(ctx: Context) {
     if (this._open) {
       return;
     }
 
     const traceId = PublicKey.random().toHex();
-    log.trace('dxos.sdk.client-services-host.open', trace.begin({ id: traceId }));
+    log.trace('dxos.client-services.host.open', trace.begin({ id: traceId }));
 
     invariant(this._config, 'config not set');
     invariant(this._storage, 'storage not set');
@@ -207,13 +224,11 @@ export class ClientServicesHost {
     invariant(this._networkManager, 'network manager not set');
 
     this._opening = true;
-    log('opening...', { lockKey: this._resourceLock?.lockKey });
+    log.info('opening...', { lockKey: this._resourceLock?.lockKey });
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
 
-    // TODO(wittjosiah): Make re-entrant.
-    // TODO(burdon): Break into components.
     this._serviceContext = new ServiceContext(
       this._storage,
       this._networkManager,
@@ -221,7 +236,6 @@ export class ClientServicesHost {
       this._modelFactory,
     );
 
-    // TODO(burdon): Start to think of DMG (dynamic services).
     this._serviceRegistry.setServices({
       SystemService: this._systemService,
 
@@ -248,6 +262,7 @@ export class ClientServicesHost {
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
       LoggingService: this._loggingService,
+      TracingService: this._tracingService,
 
       // TODO(burdon): Move to new protobuf definitions.
       DevtoolsHost: new DevtoolsServiceImpl({
@@ -257,29 +272,30 @@ export class ClientServicesHost {
       }),
     });
 
-    await this._serviceContext.open();
+    await this._serviceContext.open(ctx);
     this._opening = false;
     this._open = true;
     this._statusUpdate.emit();
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log('opened', { deviceKey });
-    log.trace('dxos.sdk.client-services-host.open', trace.end({ id: traceId }));
+    log.info('opened', { deviceKey });
+    log.trace('dxos.client-services.host.open', trace.end({ id: traceId }));
   }
 
   @synchronized
+  @Trace.span()
   async close() {
     if (!this._open) {
       return;
     }
 
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log('closing...', { deviceKey });
+    log.info('closing...', { deviceKey });
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
     await this._serviceContext.close();
     this._open = false;
     this._statusUpdate.emit();
-    log('closed', { deviceKey });
+    log.info('closed', { deviceKey });
   }
 
   async reset() {

package/src/packlets/services/service-registry.test.ts

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import { Event } from '@dxos/async';
 import { ClientServices } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols';
 import { SystemService, SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
@@ -31,12 +32,13 @@ describe('service registry', () => {
   test('builds a service registry', async () => {
     const remoteSource = 'https://remote.source';
     const serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
 
     const serviceRegistry = new ServiceRegistry(serviceBundle, {
       SystemService: new SystemServiceImpl({
         config: new Config({ runtime: { client: { remoteSource } } }),
         getCurrentStatus: () => SystemStatus.ACTIVE,
+        getDiagnostics: async () => ({}),
         onReset: () => {},
         onUpdateStatus: () => {},
         statusUpdate: new Event(),

package/src/packlets/services/util.ts

@@ -0,0 +1,33 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { PublicKey } from '@dxos/keys';
+import { humanize } from '@dxos/util';
+
+// TODO(burdon): Move to util.
+
+export type JsonStringifyOptions = {
+  truncate?: boolean;
+  humanize?: boolean;
+};
+
+export const jsonStringify = <T>(data: T, options: JsonStringifyOptions) => {
+  if (options.humanize || options.truncate) {
+    return JSON.parse(JSON.stringify(data, jsonKeyReplacer(options)));
+  }
+
+  return data;
+};
+
+const jsonKeyReplacer = (options: JsonStringifyOptions) => (key: string, value: any) => {
+  if (typeof value === 'string') {
+    // Infer if key.
+    const key = PublicKey.fromHex(value);
+    if (key.toHex() === value) {
+      return options.humanize ? humanize(key) : key.truncate();
+    }
+  }
+
+  return value;
+};

package/src/packlets/spaces/data-space.ts

@@ -55,6 +55,12 @@ export type DataSpaceParams = {
   cache?: SpaceCache;
 };
 
+/**
+ * Delete feed blocks after an epoch is created.
+ */
+// TODO(dmaretskyi): Disabled till better times https://github.com/dxos/dxos/issues/3949.
+const ENABLE_FEED_PURGE = false;
+
 @trackLeaks('open', 'close')
 export class DataSpace {
   private _ctx = new Context();
@@ -145,7 +151,7 @@ export class DataSpace {
   private async _open() {
     await this._notarizationPlugin.open();
     await this._inner.spaceState.addCredentialProcessor(this._notarizationPlugin);
-    await this._inner.open();
+    await this._inner.open(new Context());
     this._state = SpaceState.CONTROL_ONLY;
     this.stateUpdate.emit();
     this.metrics = {};
@@ -323,10 +329,16 @@ export class DataSpace {
     });
 
     await this.inner.controlPipeline.state.waitUntilTimeframe(new Timeframe([[receipt.feedKey, receipt.seq]]));
-    for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
-      const indexBeforeEpoch = epoch.timeframe.get(feed.key);
-      if (indexBeforeEpoch) {
-        await feed.clear(0, indexBeforeEpoch + 1);
+
+    // Clear feed blocks before epoch.
+    if (ENABLE_FEED_PURGE) {
+      for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
+        const indexBeforeEpoch = epoch.timeframe.get(feed.key);
+        if (indexBeforeEpoch === undefined) {
+          continue;
+        }
+
+        await feed.safeClear(0, indexBeforeEpoch + 1);
       }
     }
   }

package/src/packlets/spaces/spaces-service.test.ts

@@ -6,6 +6,7 @@ import chai, { expect } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { Space, SpacesService } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
@@ -22,7 +23,7 @@ describe('SpacesService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     spacesService = new SpacesServiceImpl(
       serviceContext.identityManager,
       serviceContext.spaceManager,

package/src/packlets/system/system-service.test.ts

@@ -35,6 +35,7 @@ describe('SystemService', () => {
       config,
       statusUpdate,
       getCurrentStatus: () => currentStatus,
+      getDiagnostics: async () => ({}),
       onUpdateStatus: (status) => {
         updateStatus.wake(status);
       },

package/src/packlets/system/system-service.ts

@@ -6,18 +6,22 @@ import { Event } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
 import { Config } from '@dxos/config';
 import {
+  GetDiagnosticsRequest,
   SystemService,
   SystemStatus,
   UpdateStatusRequest,
   QueryStatusRequest,
   QueryStatusResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
-import { MaybePromise } from '@dxos/util';
+import { jsonKeyReplacer, MaybePromise } from '@dxos/util';
+
+import { Diagnostics } from '../services';
 
 export type SystemServiceOptions = {
   config?: Config;
   statusUpdate: Event<void>;
   getCurrentStatus: () => SystemStatus;
+  getDiagnostics: () => Promise<Partial<Diagnostics>>;
   onUpdateStatus: (status: SystemStatus) => MaybePromise<void>;
   onReset: () => MaybePromise<void>;
 };
@@ -28,11 +32,20 @@ export class SystemServiceImpl implements SystemService {
   private readonly _getCurrentStatus: SystemServiceOptions['getCurrentStatus'];
   private readonly _onUpdateStatus: SystemServiceOptions['onUpdateStatus'];
   private readonly _onReset: SystemServiceOptions['onReset'];
+  private readonly _getDiagnostics: SystemServiceOptions['getDiagnostics'];
 
-  constructor({ config, statusUpdate, onUpdateStatus, getCurrentStatus, onReset }: SystemServiceOptions) {
+  constructor({
+    config,
+    statusUpdate,
+    getDiagnostics,
+    onUpdateStatus,
+    getCurrentStatus,
+    onReset,
+  }: SystemServiceOptions) {
     this._config = config;
     this._statusUpdate = statusUpdate;
     this._getCurrentStatus = getCurrentStatus;
+    this._getDiagnostics = getDiagnostics;
     this._onUpdateStatus = onUpdateStatus;
     this._onReset = onReset;
   }
@@ -41,6 +54,25 @@ export class SystemServiceImpl implements SystemService {
     return this._config?.values ?? {};
   }
 
+  /**
+   * NOTE: Since this is serialized as a JSON object, we allow the option to serialize keys.
+   */
+  async getDiagnostics({ keys }: GetDiagnosticsRequest = {}) {
+    const diagnostics = await this._getDiagnostics();
+    return {
+      timestamp: new Date(),
+      diagnostics: JSON.parse(
+        JSON.stringify(
+          diagnostics,
+          jsonKeyReplacer({
+            truncate: keys === GetDiagnosticsRequest.KEY_OPTION.TRUNCATE,
+            humanize: keys === GetDiagnosticsRequest.KEY_OPTION.HUMANIZE,
+          }),
+        ),
+      ),
+    };
+  }
+
   async updateStatus({ status }: UpdateStatusRequest) {
     await this._onUpdateStatus(status);
   }

package/src/packlets/testing/test-builder.ts

@@ -60,7 +60,7 @@ export const createPeers = async (numPeers: number) => {
   return await Promise.all(
     Array.from(Array(numPeers)).map(async () => {
       const peer = createServiceContext({ signalContext });
-      await peer.open();
+      await peer.open(new Context());
       return peer;
     }),
   );

package/src/packlets/vault/iframe-host-runtime.ts

@@ -5,6 +5,7 @@
 import { Trigger } from '@dxos/async';
 import { PROXY_CONNECTION_TIMEOUT, ShellRuntime } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log, logInfo } from '@dxos/log';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
 import { createWebRTCTransportFactory, TransportFactory } from '@dxos/network-manager';
@@ -108,7 +109,7 @@ export class IFrameHostRuntime {
         ...middleware,
       });
 
-      await Promise.all([this._clientServices.open(), this._clientRpc.open(), this._shellRuntime?.open()]);
+      await Promise.all([this._clientServices.open(new Context()), this._clientRpc.open(), this._shellRuntime?.open()]);
       this._ready.wake(undefined);
       log('started');
     } catch (err: any) {

package/src/packlets/vault/worker-runtime.ts

@@ -4,6 +4,7 @@
 
 import { Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
 import { WebRTCTransportProxyFactory } from '@dxos/network-manager';
@@ -57,7 +58,7 @@ export class WorkerRuntime {
         transportFactory: this._transportFactory,
       });
 
-      await this._clientServices.open();
+      await this._clientServices.open(new Context());
       this._ready.wake(undefined);
       log('started');
     } catch (err: any) {

package/src/packlets/vault/worker-session.ts

@@ -73,6 +73,7 @@ export class WorkerSession {
       port: appPort,
       ...middleware,
     });
+
     this._shellClientRpc = new ClientRpcServer({
       serviceRegistry: this._serviceHost.serviceRegistry,
       port: shellPort,
@@ -102,7 +103,7 @@ export class WorkerSession {
         },
       },
       port: systemPort,
-      timeout: 1000, // With low timeout heartbeat may fail if the tab's thread is saturated.
+      timeout: 1_000, // With low timeout heartbeat may fail if the tab's thread is saturated.
     });
 
     this.bridgeService = this._iframeRpc.rpc.BridgeService;
@@ -112,10 +113,14 @@ export class WorkerSession {
     log.info('opening...');
     await Promise.all([this._clientRpc.open(), this._iframeRpc.open(), this._maybeOpenShell()]);
 
+    // Wait until the worker's RPC service has started.
     await this._startTrigger.wait({ timeout: PROXY_CONNECTION_TIMEOUT });
+
+    // TODO(burdon): Comment required.
     if (this.lockKey) {
       void this._afterLockReleases(this.lockKey, () => this.close());
     }
+
     log.info('opened');
   }
 

package/src/version.ts

@@ -0,0 +1 @@
+export const DXOS_VERSION = "0.1.56-main.c19b7cd";