@dxos/client-services 0.1.56-main.5ff23aa → 0.1.56-main.6293c6e

package/src/packlets/identity/identity.ts

@@ -4,6 +4,7 @@
 
 import { Event } from '@dxos/async';
 import { AUTH_TIMEOUT, LOAD_CONTROL_FEEDS_TIMEOUT } from '@dxos/client-protocol';
+import { Context } from '@dxos/context';
 import {
   DeviceStateMachine,
   CredentialSigner,
@@ -21,6 +22,7 @@ import { log } from '@dxos/log';
 import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { DeviceAdmissionRequest } from '@dxos/protocols/proto/dxos/halo/invitations';
+import { trace } from '@dxos/tracing';
 import { ComplexSet } from '@dxos/util';
 
 import { TrustedKeySetAuthVerifier } from './authenticator';
@@ -35,6 +37,7 @@ export type IdentityParams = {
 /**
  * Agent identity manager, which includes the agent's Halo space.
  */
+@trace.resource()
 export class Identity {
   public readonly space: Space;
   private readonly _signer: Signer;
@@ -76,13 +79,15 @@ export class Identity {
     return this._deviceStateMachine.authorizedDeviceKeys;
   }
 
-  async open() {
+  @trace.span()
+  async open(ctx: Context) {
     await this.space.spaceState.addCredentialProcessor(this._deviceStateMachine);
     await this.space.spaceState.addCredentialProcessor(this._profileStateMachine);
-    await this.space.open();
+    await this.space.open(ctx);
   }
 
-  async close() {
+  @trace.span()
+  async close(ctx: Context) {
     await this.authVerifier.close();
     await this.space.spaceState.removeCredentialProcessor(this._profileStateMachine);
     await this.space.spaceState.removeCredentialProcessor(this._deviceStateMachine);

package/src/packlets/invitations/device-invitation-protocol.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { asyncChain } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test, afterTest } from '@dxos/test';
 
@@ -19,7 +20,7 @@ const closeAfterTest = async (peer: ServiceContext) => {
 describe('services/device', () => {
   test('creates identity', async () => {
     const peer = createServiceContext();
-    await peer.open();
+    await peer.open(new Context());
     afterTest(() => peer.close());
 
     const identity = await peer.createIdentity();

package/src/packlets/network/network-service.test.ts

@@ -5,6 +5,7 @@
 import { expect } from 'chai';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { NetworkService, ConnectionState } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
 
@@ -18,7 +19,7 @@ describe('NetworkService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     networkService = new NetworkServiceImpl(serviceContext.networkManager, serviceContext.signalManager);
   });
 

package/src/packlets/services/client-rpc-server.ts

@@ -5,7 +5,7 @@
 import { ClientServices } from '@dxos/client-protocol';
 import { Any, ServiceHandler, Stream } from '@dxos/codec-protobuf';
 import { raise } from '@dxos/debug';
-import { parseMethodName, RpcPeer, RpcPeerOptions } from '@dxos/rpc';
+import { parseMethodName, RpcPeer, RpcPeerOptions, ServiceBundle } from '@dxos/rpc';
 import { MaybePromise } from '@dxos/util';
 
 import { ServiceRegistry } from './service-registry';
@@ -74,16 +74,16 @@ export class ClientRpcServer {
   private _getServiceHandler(serviceName: string) {
     if (!this._handlerCache.has(serviceName)) {
       const [key, descriptor] =
-        Object.entries(this._serviceRegistry.descriptors).find(
+        Object.entries(this._serviceRegistry.descriptors as ServiceBundle<Record<string, any>>).find(
           ([key, descriptor]) => descriptor.name === serviceName,
         ) ?? raise(new Error(`Service not available: ${serviceName}`));
 
-      const service = this._serviceRegistry.services[key as keyof ClientServices];
+      const service = this._serviceRegistry.services[key as keyof ClientServices] as any;
       if (!service) {
         throw new Error(`Service not available: ${serviceName}`);
       }
 
-      this._handlerCache.set(serviceName, descriptor.createServer(service as any));
+      this._handlerCache.set(serviceName, descriptor.createServer(service));
     }
 
     return this._handlerCache.get(serviceName)!;

package/src/packlets/services/service-context.test.ts

@@ -2,9 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
+import { expect } from 'chai';
+
+import { DocumentModel, DocumentModelState, MutationBuilder } from '@dxos/document-model';
+import { createModelMutation, encodeModelMutation, genesisMutation } from '@dxos/echo-db';
+import { WriteReceipt } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { describe, test } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
 
 import { createServiceContext, syncItemsLocal } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -16,11 +25,61 @@ describe('services/ServiceContext', () => {
     await device1.createIdentity();
     const space1 = await device1.dataSpaceManager!.createSpace();
 
+    const itemId = PublicKey.random().toHex();
+    await space1.dataPipeline.databaseHost!.getWriteStream()?.write({
+      batch: genesisMutation(itemId, DocumentModel.meta.type),
+    });
+
+    let counter = 0;
+
+    for (const _ in range(5)) {
+      const receipts: WriteReceipt[] = [];
+      for (const _ in range(50)) {
+        receipts.push(
+          await space1.dataPipeline.databaseHost!.getWriteStream()!.write({
+            batch: createModelMutation(
+              itemId,
+              encodeModelMutation(DocumentModel.meta, new MutationBuilder().set('counter', ++counter).build()),
+            ),
+          }),
+        );
+      }
+
+      await space1.dataPipeline.pipelineState!.waitUntilTimeframe(
+        Timeframe.merge(...receipts.map((receipt) => new Timeframe([[receipt.feedKey, receipt.seq]]))),
+      );
+
+      await space1.createEpoch();
+      log.info('epoch', { number: space1.dataPipeline.currentEpoch?.subject.assertion.number });
+    }
+
     const device2 = createServiceContext({ signalContext: networkContext });
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     await device2.dataSpaceManager!.waitUntilSpaceReady(space1!.key);
     const space2 = await device2.dataSpaceManager!.spaces.get(space1.key);
+
+    log.info('peer 2', {
+      currentEpoch: space2!.dataPipeline.currentEpoch?.subject.assertion.number,
+      appliedEpoch: space2!.dataPipeline.appliedEpoch?.subject.assertion.number,
+    });
+
+    await space2?.dataPipeline.onNewEpoch.waitForCondition(
+      () =>
+        space2!.dataPipeline.appliedEpoch?.subject.assertion.number ===
+        space1.dataPipeline.currentEpoch?.subject.assertion.number,
+    );
+
+    expect(space2!.dataPipeline.currentEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.currentEpoch!.subject.assertion.number,
+    );
+    expect(space2!.dataPipeline.appliedEpoch!.subject.assertion.number).to.equal(
+      space1.dataPipeline.appliedEpoch!.subject.assertion.number,
+    );
+
+    const item2 = space2!.dataPipeline.itemManager.entities.get(itemId);
+    expect((item2!.state as DocumentModelState).data.counter).to.equal(counter);
+
     await syncItemsLocal(space1.dataPipeline, space2!.dataPipeline);
   });
 

package/src/packlets/services/service-context.ts

@@ -3,6 +3,7 @@
 //
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { CredentialProcessor, getCredentialAssertion } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import {
@@ -26,6 +27,8 @@ import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Storage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { trace as Trace } from '@dxos/tracing';
+import { safeInstanceof } from '@dxos/util';
 
 import { CreateIdentityOptions, IdentityManager, JoinIdentityParams } from '../identity';
 import {
@@ -40,6 +43,9 @@ import { DataSpaceManager, SigningContext } from '../spaces';
  * Shared backend for all client services.
  */
 // TODO(burdon): Rename/break-up into smaller components. And/or make members private.
+// TODO(dmaretskyi): Gets duplicated in CJS build between normal and testing bundles.
+@safeInstanceof('dxos.client-services.ServiceContext')
+@Trace.resource()
 export class ServiceContext {
   public readonly initialized = new Trigger();
   public readonly dataServiceSubscriptions = new DataServiceSubscriptions();
@@ -121,7 +127,8 @@ export class ServiceContext {
     );
   }
 
-  async open() {
+  @Trace.span()
+  async open(ctx: Context) {
     await this._checkStorageVersion();
 
     log('opening...');
@@ -129,9 +136,9 @@ export class ServiceContext {
     await this.signalManager.open();
     await this.networkManager.open();
     await this.spaceManager.open();
-    await this.identityManager.open();
+    await this.identityManager.open(ctx);
     if (this.identityManager.identity) {
-      await this._initialize();
+      await this._initialize(ctx);
     }
     log.trace('dxos.sdk.service-context.open', trace.end({ id: this._instanceId }));
     log('opened');
@@ -155,7 +162,7 @@ export class ServiceContext {
   async createIdentity(params: CreateIdentityOptions = {}) {
     const identity = await this.identityManager.createIdentity(params);
 
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -168,7 +175,7 @@ export class ServiceContext {
   private async _acceptIdentity(params: JoinIdentityParams) {
     const identity = await this.identityManager.acceptIdentity(params);
 
-    await this._initialize();
+    await this._initialize(new Context());
     return identity;
   }
 
@@ -181,7 +188,8 @@ export class ServiceContext {
   }
 
   // Called when identity is created.
-  private async _initialize() {
+  @Trace.span()
+  private async _initialize(ctx: Context) {
     log('initializing spaces...');
     const identity = this.identityManager.identity ?? failUndefined();
     const signingContext: SigningContext = {

package/src/packlets/services/service-host.test.ts

@@ -8,6 +8,7 @@ import { rmSync } from 'node:fs';
 
 import { asyncTimeout, latch, Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { verifyPresentation } from '@dxos/credentials';
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManagerContext } from '@dxos/messaging';
@@ -30,7 +31,7 @@ describe('ClientServicesHost', () => {
 
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
@@ -49,7 +50,7 @@ describe('ClientServicesHost', () => {
 
   test('write and query credentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
@@ -86,7 +87,7 @@ describe('ClientServicesHost', () => {
 
   test('sign presentation', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
-    await host.open();
+    await host.open(new Context());
     afterTest(() => host.close());
 
     await host.services.IdentityService!.createIdentity({});
@@ -117,7 +118,7 @@ describe('ClientServicesHost', () => {
     });
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
 
       await host.services.IdentityService?.createIdentity({});
 
@@ -129,7 +130,7 @@ describe('ClientServicesHost', () => {
 
     {
       const host = createServiceHost(config, new MemorySignalManagerContext());
-      await host.open();
+      await host.open(new Context());
       const trigger = new Trigger<Identity>();
 
       const stream = host.services.IdentityService?.queryIdentity();

package/src/packlets/services/service-host.ts

@@ -5,6 +5,7 @@
 import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, ClientServices } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { DocumentModel } from '@dxos/document-model';
 import { DataServiceImpl } from '@dxos/echo-pipeline';
 import { invariant } from '@dxos/invariant';
@@ -17,6 +18,7 @@ import { trace } from '@dxos/protocols';
 import { SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
 import { Storage } from '@dxos/random-access-storage';
 import { TextModel } from '@dxos/text-model';
+import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
 
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsServiceImpl, DevtoolsHostEvents } from '../devtools';
@@ -65,11 +67,13 @@ export type InitializeOptions = {
 /**
  * Remote service implementation.
  */
+@Trace.resource()
 export class ClientServicesHost {
   private readonly _resourceLock?: ResourceLock;
   private readonly _serviceRegistry: ServiceRegistry<ClientServices>;
   private readonly _systemService: SystemServiceImpl;
   private readonly _loggingService: LoggingServiceImpl;
+  private readonly _tracingService = TRACE_PROCESSOR.createTraceSender();
 
   private _config?: Config;
   private readonly _statusUpdate = new Event<void>();
@@ -80,7 +84,11 @@ export class ClientServicesHost {
   private _callbacks?: ClientServicesHostCallbacks;
 
   private _serviceContext!: ServiceContext;
+
+  @Trace.info()
   private _opening = false;
+
+  @Trace.info()
   private _open = false;
 
   constructor({
@@ -106,7 +114,7 @@ export class ClientServicesHost {
         lockKey,
         onAcquire: () => {
           if (!this._opening) {
-            void this.open();
+            void this.open(new Context());
           }
         },
         onRelease: () => this.close(),
@@ -136,6 +144,7 @@ export class ClientServicesHost {
 
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
       SystemService: this._systemService,
+      TracingService: this._tracingService,
     });
   }
 
@@ -170,7 +179,7 @@ export class ClientServicesHost {
    */
   initialize({ config, ...options }: InitializeOptions) {
     invariant(!this._open, 'service host is open');
-    log.info('initializing...');
+    log('initializing...');
 
     if (config) {
       invariant(!this._config, 'config already set');
@@ -196,11 +205,12 @@ export class ClientServicesHost {
       signalManager,
     });
 
-    log.info('initialized');
+    log('initialized');
   }
 
   @synchronized
-  async open() {
+  @Trace.span()
+  async open(ctx: Context) {
     if (this._open) {
       return;
     }
@@ -214,7 +224,7 @@ export class ClientServicesHost {
     invariant(this._networkManager, 'network manager not set');
 
     this._opening = true;
-    log.info('opening...', { lockKey: this._resourceLock?.lockKey });
+    log('opening...', { lockKey: this._resourceLock?.lockKey });
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
@@ -229,7 +239,11 @@ export class ClientServicesHost {
     this._serviceRegistry.setServices({
       SystemService: this._systemService,
 
-      IdentityService: new IdentityServiceImpl(this._serviceContext),
+      IdentityService: new IdentityServiceImpl(
+        (params) => this._serviceContext.createIdentity(params),
+        this._serviceContext.identityManager,
+        this._serviceContext.keyring,
+      ),
 
       InvitationsService: new InvitationsServiceImpl(this._serviceContext.invitations, (invitation) =>
         this._serviceContext.getInvitationHandler(invitation),
@@ -252,6 +266,7 @@ export class ClientServicesHost {
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
       LoggingService: this._loggingService,
+      TracingService: this._tracingService,
 
       // TODO(burdon): Move to new protobuf definitions.
       DevtoolsHost: new DevtoolsServiceImpl({
@@ -261,29 +276,30 @@ export class ClientServicesHost {
       }),
     });
 
-    await this._serviceContext.open();
+    await this._serviceContext.open(ctx);
     this._opening = false;
     this._open = true;
     this._statusUpdate.emit();
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log.info('opened', { deviceKey });
+    log('opened', { deviceKey });
     log.trace('dxos.client-services.host.open', trace.end({ id: traceId }));
   }
 
   @synchronized
+  @Trace.span()
   async close() {
     if (!this._open) {
       return;
     }
 
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
-    log.info('closing...', { deviceKey });
+    log('closing...', { deviceKey });
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
     await this._serviceContext.close();
     this._open = false;
     this._statusUpdate.emit();
-    log.info('closed', { deviceKey });
+    log('closed', { deviceKey });
   }
 
   async reset() {

package/src/packlets/services/service-registry.test.ts

@@ -7,6 +7,7 @@ import { expect } from 'chai';
 import { Event } from '@dxos/async';
 import { ClientServices } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { schema } from '@dxos/protocols';
 import { SystemService, SystemStatus } from '@dxos/protocols/proto/dxos/client/services';
@@ -31,7 +32,7 @@ describe('service registry', () => {
   test('builds a service registry', async () => {
     const remoteSource = 'https://remote.source';
     const serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
 
     const serviceRegistry = new ServiceRegistry(serviceBundle, {
       SystemService: new SystemServiceImpl({

package/src/packlets/spaces/data-space.ts

@@ -55,6 +55,12 @@ export type DataSpaceParams = {
   cache?: SpaceCache;
 };
 
+/**
+ * Delete feed blocks after an epoch is created.
+ */
+// TODO(dmaretskyi): Disabled till better times https://github.com/dxos/dxos/issues/3949.
+const ENABLE_FEED_PURGE = false;
+
 @trackLeaks('open', 'close')
 export class DataSpace {
   private _ctx = new Context();
@@ -145,7 +151,7 @@ export class DataSpace {
   private async _open() {
     await this._notarizationPlugin.open();
     await this._inner.spaceState.addCredentialProcessor(this._notarizationPlugin);
-    await this._inner.open();
+    await this._inner.open(new Context());
     this._state = SpaceState.CONTROL_ONLY;
     this.stateUpdate.emit();
     this.metrics = {};
@@ -323,10 +329,16 @@ export class DataSpace {
     });
 
     await this.inner.controlPipeline.state.waitUntilTimeframe(new Timeframe([[receipt.feedKey, receipt.seq]]));
-    for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
-      const indexBeforeEpoch = epoch.timeframe.get(feed.key);
-      if (indexBeforeEpoch) {
-        await feed.clear(0, indexBeforeEpoch + 1);
+
+    // Clear feed blocks before epoch.
+    if (ENABLE_FEED_PURGE) {
+      for (const feed of this.inner.dataPipeline.pipelineState?.feeds ?? []) {
+        const indexBeforeEpoch = epoch.timeframe.get(feed.key);
+        if (indexBeforeEpoch === undefined) {
+          continue;
+        }
+
+        await feed.safeClear(0, indexBeforeEpoch + 1);
       }
     }
   }

package/src/packlets/spaces/spaces-service.test.ts

@@ -6,6 +6,7 @@ import chai, { expect } from 'chai';
 import chaiAsPromised from 'chai-as-promised';
 
 import { Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { Space, SpacesService } from '@dxos/protocols/proto/dxos/client/services';
 import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
@@ -22,7 +23,7 @@ describe('SpacesService', () => {
 
   beforeEach(async () => {
     serviceContext = createServiceContext();
-    await serviceContext.open();
+    await serviceContext.open(new Context());
     spacesService = new SpacesServiceImpl(
       serviceContext.identityManager,
       serviceContext.spaceManager,

package/src/packlets/testing/test-builder.ts

@@ -60,7 +60,7 @@ export const createPeers = async (numPeers: number) => {
   return await Promise.all(
     Array.from(Array(numPeers)).map(async () => {
       const peer = createServiceContext({ signalContext });
-      await peer.open();
+      await peer.open(new Context());
       return peer;
     }),
   );

package/src/packlets/vault/iframe-host-runtime.ts

@@ -5,6 +5,7 @@
 import { Trigger } from '@dxos/async';
 import { PROXY_CONNECTION_TIMEOUT, ShellRuntime } from '@dxos/client-protocol';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log, logInfo } from '@dxos/log';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
 import { createWebRTCTransportFactory, TransportFactory } from '@dxos/network-manager';
@@ -108,7 +109,7 @@ export class IFrameHostRuntime {
         ...middleware,
       });
 
-      await Promise.all([this._clientServices.open(), this._clientRpc.open(), this._shellRuntime?.open()]);
+      await Promise.all([this._clientServices.open(new Context()), this._clientRpc.open(), this._shellRuntime?.open()]);
       this._ready.wake(undefined);
       log('started');
     } catch (err: any) {

package/src/packlets/vault/worker-runtime.ts

@@ -4,6 +4,7 @@
 
 import { Trigger } from '@dxos/async';
 import { Config } from '@dxos/config';
+import { Context } from '@dxos/context';
 import { log } from '@dxos/log';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
 import { WebRTCTransportProxyFactory } from '@dxos/network-manager';
@@ -57,7 +58,7 @@ export class WorkerRuntime {
         transportFactory: this._transportFactory,
       });
 
-      await this._clientServices.open();
+      await this._clientServices.open(new Context());
       this._ready.wake(undefined);
       log('started');
     } catch (err: any) {

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.1.56-main.5ff23aa";
+export const DXOS_VERSION = "0.1.56-main.6293c6e";