@dxos/client-services 0.4.10-main.d4e372f → 0.4.10-main.d51f2c2

package/src/packlets/services/automerge-host.test.ts

@@ -6,8 +6,8 @@ import { expect } from 'chai';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { AutomergeHost, DataServiceImpl } from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { AutomergeContext } from '@dxos/echo-schema';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, test } from '@dxos/test';
 
 describe('AutomergeHost', () => {
@@ -19,10 +19,16 @@ describe('AutomergeHost', () => {
     // creates repo and document | replicates repo | finds document in repo
     //
 
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
     afterTest(() => host.close());
+
     const dataService = new DataServiceImpl(host);
     const client = new AutomergeContext(dataService);
     afterTest(() => client.close());

package/src/packlets/services/index.ts

@@ -3,7 +3,7 @@
 //
 
 export * from './client-rpc-server';
-export * from './diagnostics';
 export * from './service-context';
 export * from './service-host';
 export * from './service-registry';
+export { ClientServicesProviderResource } from './util';

package/src/packlets/services/service-context.test.ts

@@ -4,7 +4,7 @@
 
 import { MemorySignalManagerContext } from '@dxos/messaging';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
-import { describe, test } from '@dxos/test';
+import { describe, openAndClose, test } from '@dxos/test';
 
 import { createServiceContext } from '../testing';
 import { performInvitation } from '../testing/invitation-utils';
@@ -12,10 +12,10 @@ import { performInvitation } from '../testing/invitation-utils';
 describe('services/ServiceContext', () => {
   test('new space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
-    const device1 = createServiceContext({ signalContext: networkContext });
+    const device1 = await createServiceContext({ signalContext: networkContext });
     await device1.createIdentity();
 
-    const device2 = createServiceContext({ signalContext: networkContext });
+    const device2 = await createServiceContext({ signalContext: networkContext });
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
     const space1 = await device1.dataSpaceManager!.createSpace();
@@ -26,13 +26,16 @@ describe('services/ServiceContext', () => {
 
   test('joined space is synchronized on device invitations', async () => {
     const networkContext = new MemorySignalManagerContext();
-    const device1 = createServiceContext({ signalContext: networkContext });
+    const device1 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device1.automergeHost);
     await device1.createIdentity();
 
-    const device2 = createServiceContext({ signalContext: networkContext });
+    const device2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(device2.automergeHost);
     await Promise.all(performInvitation({ host: device1, guest: device2, options: { kind: Invitation.Kind.DEVICE } }));
 
-    const identity2 = createServiceContext({ signalContext: networkContext });
+    const identity2 = await createServiceContext({ signalContext: networkContext });
+    await openAndClose(identity2.automergeHost);
     await identity2.createIdentity();
     const space1 = await identity2.dataSpaceManager!.createSpace();
     await Promise.all(

package/src/packlets/services/service-context.ts

@@ -2,8 +2,10 @@
 // Copyright 2022 DXOS.org
 //
 
+import { type Level } from 'level';
+
 import { Trigger } from '@dxos/async';
-import { Context } from '@dxos/context';
+import { Context, Resource } from '@dxos/context';
 import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
@@ -30,7 +32,7 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
-import { createGetAllDocuments, createLoadDocuments } from '../indexing';
+import { createDocumentsIterator, createSelectedDocumentsIterator } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -47,7 +49,7 @@ export type ServiceContextRuntimeParams = IdentityManagerRuntimeParams & DataSpa
 // TODO(dmaretskyi): Gets duplicated in CJS build between normal and testing bundles.
 @safeInstanceof('dxos.client-services.ServiceContext')
 @Trace.resource()
-export class ServiceContext {
+export class ServiceContext extends Resource {
   public readonly initialized = new Trigger();
   public readonly metadataStore: MetadataStore;
   /**
@@ -78,10 +80,13 @@ export class ServiceContext {
 
   constructor(
     public readonly storage: Storage,
+    public readonly level: Level<string, string>,
     public readonly networkManager: NetworkManager,
     public readonly signalManager: SignalManager,
     public readonly _runtimeParams?: IdentityManagerRuntimeParams & DataSpaceManagerRuntimeParams,
   ) {
+    super();
+
     // TODO(burdon): Move strings to constants.
     this.metadataStore = new MetadataStore(storage.createDirectory('metadata'));
     this.snapshotStore = new SnapshotStore(storage.createDirectory('snapshots'));
@@ -115,18 +120,19 @@ export class ServiceContext {
       this._runtimeParams as IdentityManagerRuntimeParams,
     );
 
-    this.indexMetadata = new IndexMetadataStore({ directory: storage.createDirectory('index-metadata') });
+    this.indexMetadata = new IndexMetadataStore({ db: level.sublevel('index-metadata') });
 
     this.automergeHost = new AutomergeHost({
       directory: storage.createDirectory('automerge'),
+      db: level.sublevel('automerge'),
       metadata: this.indexMetadata,
     });
 
     this.indexer = new Indexer({
       indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
       metadataStore: this.indexMetadata,
-      loadDocuments: createLoadDocuments(this.automergeHost),
-      getAllDocuments: createGetAllDocuments(this.automergeHost),
+      loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
+      getAllDocuments: createDocumentsIterator(this.automergeHost),
     });
 
     this.invitations = new InvitationsHandler(this.networkManager);
@@ -145,7 +151,7 @@ export class ServiceContext {
   }
 
   @Trace.span()
-  async open(ctx: Context) {
+  protected override async _open(ctx: Context) {
     await this._checkStorageVersion();
 
     log('opening...');
@@ -153,6 +159,7 @@ export class ServiceContext {
     await this.signalManager.open();
     await this.networkManager.open();
 
+    await this.automergeHost.open();
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);
@@ -163,7 +170,7 @@ export class ServiceContext {
     log('opened');
   }
 
-  async close() {
+  protected override async _close() {
     log('closing...');
     if (this._deviceSpaceSync && this.identityManager.identity) {
       await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);

package/src/packlets/services/service-host.ts

@@ -2,12 +2,20 @@
 // Copyright 2021 DXOS.org
 //
 
+import { type Level } from 'level';
+
 import { Event, synchronized } from '@dxos/async';
-import { Properties, clientServiceBundle, defaultKey, type ClientServices } from '@dxos/client-protocol';
+import { clientServiceBundle, defaultKey, type ClientServices, Properties } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { DataServiceImpl, type SpaceDoc } from '@dxos/echo-pipeline';
-import { getAutomergeObjectCore, getRawDoc, type TypedObject } from '@dxos/echo-schema';
+import {
+  DataServiceImpl,
+  type ObjectStructure,
+  encodeReference,
+  type SpaceDoc,
+  type MyLevel,
+} from '@dxos/echo-pipeline';
+import { getTypeReference } from '@dxos/echo-schema';
 import { IndexServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
@@ -21,18 +29,22 @@ import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
 import { assignDeep } from '@dxos/util';
 import { WebsocketRpcClient } from '@dxos/websocket-rpc';
 
-import { createDiagnostics } from './diagnostics';
 import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
 import { ServiceRegistry } from './service-registry';
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsHostEvents, DevtoolsServiceImpl } from '../devtools';
+import {
+  type CollectDiagnosticsBroadcastHandler,
+  createCollectDiagnosticsBroadcastHandler,
+  createDiagnostics,
+} from '../diagnostics';
 import { IdentityServiceImpl, type CreateIdentityOptions } from '../identity';
 import { InvitationsServiceImpl } from '../invitations';
 import { Lock, type ResourceLock } from '../locks';
 import { LoggingServiceImpl } from '../logging';
 import { NetworkServiceImpl } from '../network';
 import { SpacesServiceImpl } from '../spaces';
-import { createStorageObjects } from '../storage';
+import { createLevel, createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
 
 export type ClientServicesHostParams = {
@@ -44,6 +56,7 @@ export type ClientServicesHostParams = {
   signalManager?: SignalManager;
   connectionLog?: boolean;
   storage?: Storage;
+  level?: MyLevel;
   lockKey?: string;
   callbacks?: ClientServicesHostCallbacks;
   runtimeParams?: ServiceContextRuntimeParams;
@@ -76,11 +89,13 @@ export class ClientServicesHost {
   private _signalManager?: SignalManager;
   private _networkManager?: NetworkManager;
   private _storage?: Storage;
+  private _level?: Level<string, string>;
   private _callbacks?: ClientServicesHostCallbacks;
   private _devtoolsProxy?: WebsocketRpcClient<{}, ClientServices>;
 
   private _serviceContext!: ServiceContext;
   private readonly _runtimeParams?: ServiceContextRuntimeParams;
+  private diagnosticsBroadcastHandler: CollectDiagnosticsBroadcastHandler;
 
   @Trace.info()
   private _opening = false;
@@ -93,12 +108,14 @@ export class ClientServicesHost {
     transportFactory,
     signalManager,
     storage,
+    level,
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
     callbacks,
     runtimeParams,
   }: ClientServicesHostParams = {}) {
     this._storage = storage;
+    this._level = level;
     this._callbacks = callbacks;
     this._runtimeParams = runtimeParams;
 
@@ -138,6 +155,7 @@ export class ClientServicesHost {
       },
     });
 
+    this.diagnosticsBroadcastHandler = createCollectDiagnosticsBroadcastHandler(this._systemService);
     this._loggingService = new LoggingServiceImpl();
 
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
@@ -226,12 +244,19 @@ export class ClientServicesHost {
 
     this._opening = true;
     log('opening...', { lockKey: this._resourceLock?.lockKey });
+
+    if (!this._level) {
+      this._level = await createLevel(this._config.get('runtime.client.storage', {})!);
+    }
+    await this._level.open();
+
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
 
     this._serviceContext = new ServiceContext(
       this._storage,
+      this._level,
       this._networkManager,
       this._signalManager,
       this._runtimeParams,
@@ -301,6 +326,7 @@ export class ClientServicesHost {
       });
       void this._devtoolsProxy.open();
     }
+    this.diagnosticsBroadcastHandler.start();
 
     this._opening = false;
     this._open = true;
@@ -319,10 +345,12 @@ export class ClientServicesHost {
 
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
     log('closing...', { deviceKey });
+    this.diagnosticsBroadcastHandler.stop();
     await this._devtoolsProxy?.close();
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
     await this._serviceContext.close();
+    await this._level?.close();
     this._open = false;
     this._statusUpdate.emit();
     log('closed', { deviceKey });
@@ -347,18 +375,30 @@ export class ClientServicesHost {
     await this._serviceContext.initialized.wait();
     const space = await this._serviceContext.dataSpaceManager!.createSpace();
 
-    const obj: TypedObject = new Properties(undefined);
-    obj[defaultKey] = identity.identityKey.toHex();
-
     const automergeIndex = space.automergeSpaceState.rootUrl;
     invariant(automergeIndex);
     const document = await this._serviceContext.automergeHost.repo.find<SpaceDoc>(automergeIndex as any);
     await document.whenReady();
 
+    // TODO(dmaretskyi): Better API for low-level data access.
+    const properties: ObjectStructure = {
+      system: {
+        type: encodeReference(getTypeReference(Properties)!),
+      },
+      data: {
+        [defaultKey]: identity.identityKey.toHex(),
+      },
+      meta: {
+        keys: [],
+      },
+    };
+    const propertiesId = PublicKey.random().toHex();
     document.change((doc: SpaceDoc) => {
-      assignDeep(doc, ['objects', getAutomergeObjectCore(obj).id], getRawDoc(obj).handle.docSync());
+      assignDeep(doc, ['objects', propertiesId], properties);
     });
 
+    await this._serviceContext.automergeHost.repo.flush();
+
     return identity;
   }
 }

package/src/packlets/services/service-registry.test.ts

@@ -31,7 +31,7 @@ const serviceBundle = createServiceBundle<TestServices>({
 describe('service registry', () => {
   test('builds a service registry', async () => {
     const remoteSource = 'https://remote.source';
-    const serviceContext = createServiceContext();
+    const serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
 
     const serviceRegistry = new ServiceRegistry(serviceBundle, {

package/src/packlets/services/util.ts

@@ -5,6 +5,8 @@
 import { PublicKey } from '@dxos/keys';
 import { humanize } from '@dxos/util';
 
+export const ClientServicesProviderResource = Symbol.for('dxos.resource.ClientServices');
+
 // TODO(burdon): Move to util.
 
 export type JsonStringifyOptions = {

package/src/packlets/spaces/data-space-manager.test.ts

@@ -20,7 +20,7 @@ describe('DataSpaceManager', () => {
 
     const peer = builder.createPeer();
     await peer.createIdentity();
-    await openAndClose(peer.dataSpaceManager);
+    await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
     const space = await peer.dataSpaceManager.createSpace();
 
@@ -42,7 +42,7 @@ describe('DataSpaceManager', () => {
     const peer2 = builder.createPeer();
     await peer2.createIdentity();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -111,7 +111,7 @@ describe('DataSpaceManager', () => {
     await peer2.createIdentity();
     await peer2.dataSpaceManager.open();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -153,7 +153,7 @@ describe('DataSpaceManager', () => {
 
       const peer = builder.createPeer();
       await peer.createIdentity();
-      await openAndClose(peer.dataSpaceManager);
+      await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
       const space = await peer.dataSpaceManager.createSpace();
       await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);

package/src/packlets/spaces/spaces-service.test.ts

@@ -22,7 +22,7 @@ describe('SpacesService', () => {
   let spacesService: SpacesService;
 
   beforeEach(async () => {
-    serviceContext = createServiceContext();
+    serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
     spacesService = new SpacesServiceImpl(serviceContext.identityManager, serviceContext.spaceManager, async () => {
       await serviceContext.initialized.wait();

package/src/packlets/storage/index.ts

@@ -3,3 +3,4 @@
 //
 
 export * from './storage';
+export * from './level';

package/src/packlets/storage/level.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Level } from 'level';
+import path from 'node:path';
+
+import { PublicKey } from '@dxos/keys';
+import { type Runtime } from '@dxos/protocols/proto/dxos/config';
+
+import { getRootPath, isPersistent } from './util';
+
+export const createLevel = async (config: Runtime.Client.Storage) => {
+  const persistent = isPersistent(config);
+  const storagePath = persistent ? path.join(getRootPath(config), 'level') : `/tmp/dxos-${PublicKey.random().toHex()}`;
+  const level = new Level<string, string>(storagePath);
+  await level.open();
+  return level;
+};

package/src/packlets/storage/storage.ts

@@ -4,22 +4,16 @@
 // Copyright 2023 DXOS.org
 //
 
-import { DX_DATA } from '@dxos/client-protocol';
 import { InvalidConfigError } from '@dxos/protocols';
 import { Runtime } from '@dxos/protocols/proto/dxos/config';
 import { createStorage, StorageType } from '@dxos/random-access-storage';
-import { isNode } from '@dxos/util';
 
 import StorageDriver = Runtime.Client.Storage.StorageDriver;
+import { getRootPath } from './util';
 
 // TODO(burdon): Factor out.
 export const createStorageObjects = (config: Runtime.Client.Storage) => {
-  const {
-    persistent = false,
-    keyStore,
-    dataStore,
-    dataRoot = isNode() ? DX_DATA : 'dxos/storage', // TODO(burdon): Factor out const.
-  } = config ?? {};
+  const { persistent = false, keyStore, dataStore } = config ?? {};
 
   if (persistent && dataStore === StorageDriver.RAM) {
     throw new InvalidConfigError('RAM storage cannot be used in persistent mode.');
@@ -37,7 +31,7 @@ export const createStorageObjects = (config: Runtime.Client.Storage) => {
   return {
     storage: createStorage({
       type: persistent ? toStorageType(dataStore) : StorageType.RAM,
-      root: `${dataRoot}/`,
+      root: getRootPath(config),
     }),
   };
 };

package/src/packlets/storage/util.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { DX_DATA } from '@dxos/client-protocol';
+import { Runtime } from '@dxos/protocols/proto/dxos/config';
+import { isNode } from '@dxos/util';
+
+export const getRootPath = (config: Runtime.Client.Storage) => {
+  const { dataRoot = isNode() ? DX_DATA : 'dxos/storage' } = config ?? {};
+  return `${dataRoot}/`;
+};
+
+export const isPersistent = (config: Runtime.Client.Storage) => {
+  const { persistent = false } = config ?? {};
+  return (
+    (config.dataStore !== undefined && config.dataStore !== Runtime.Client.Storage.StorageDriver.RAM) || persistent
+  );
+};

package/src/packlets/system/system-service.ts

@@ -16,7 +16,7 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 import { jsonKeyReplacer, type MaybePromise } from '@dxos/util';
 
-import { type Diagnostics } from '../services';
+import { type Diagnostics } from '../diagnostics';
 import { getPlatform } from '../services/platform';
 
 export type SystemServiceOptions = {

package/src/packlets/testing/test-builder.ts

@@ -6,13 +6,22 @@ import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
 import { createCredentialSignerWithChain, CredentialGenerator } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
+import {
+  AutomergeHost,
+  MetadataStore,
+  type MyLevel,
+  SnapshotStore,
+  SpaceManager,
+  valueEncoding,
+} from '@dxos/echo-pipeline';
+import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
 import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
 import { MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
 import { createStorage, StorageType, type Storage } from '@dxos/random-access-storage';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { type MaybePromise } from '@dxos/util';
 
 import { ClientServicesHost, ServiceContext } from '../services';
 import { DataSpaceManager, type SigningContext } from '../spaces';
@@ -29,7 +38,7 @@ export const createServiceHost = (config: Config, signalManagerContext: MemorySi
   });
 };
 
-export const createServiceContext = ({
+export const createServiceContext = async ({
   signalContext = new MemorySignalManagerContext(),
   storage = createStorage({ type: StorageType.RAM }),
 }: {
@@ -41,8 +50,10 @@ export const createServiceContext = ({
     signalManager,
     transportFactory: MemoryTransportFactory,
   });
+  const level = createTestLevel();
+  await level.open();
 
-  return new ServiceContext(storage, networkManager, signalManager);
+  return new ServiceContext(storage, level, networkManager, signalManager);
 };
 
 export const createPeers = async (numPeers: number) => {
@@ -50,7 +61,7 @@ export const createPeers = async (numPeers: number) => {
 
   return await Promise.all(
     Array.from(Array(numPeers)).map(async () => {
-      const peer = createServiceContext({ signalContext });
+      const peer = await createServiceContext({ signalContext });
       await peer.open(new Context());
       return peer;
     }),
@@ -83,6 +94,7 @@ export type TestPeerOpts = {
 
 export type TestPeerProps = {
   storage?: Storage;
+  level?: MaybePromise<MyLevel>;
   feedStore?: FeedStore<any>;
   metadataStore?: MetadataStore;
   keyring?: Keyring;
@@ -115,6 +127,10 @@ export class TestPeer {
     return (this._props.keyring ??= new Keyring(this.storage.createDirectory('keyring')));
   }
 
+  get level() {
+    return (this._props.level ??= createTestLevel());
+  }
+
   get feedStore() {
     return (this._props.feedStore ??= new FeedStore({
       factory: new FeedFactory({
@@ -161,7 +177,9 @@ export class TestPeer {
   }
 
   get automergeHost() {
-    return (this._props.automergeHost ??= new AutomergeHost({ directory: this.storage.createDirectory('automerge') }));
+    return (this._props.automergeHost ??= new AutomergeHost({
+      db: Promise.resolve(this.level).then((level) => level.sublevel('automerge')),
+    }));
   }
 
   get dataSpaceManager() {

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.4.10-main.d4e372f";
+export const DXOS_VERSION = "0.4.10-main.d51f2c2";