@dxos/client-services 0.4.10-main.fa5a270 → 0.4.10-main.fd8ea31

package/src/packlets/services/service-host.ts

@@ -2,13 +2,21 @@
 // Copyright 2021 DXOS.org
 //
 
+import { type Level } from 'level';
+
 import { Event, synchronized } from '@dxos/async';
-import { Properties, clientServiceBundle, defaultKey, type ClientServices } from '@dxos/client-protocol';
+import { clientServiceBundle, defaultKey, type ClientServices, Properties } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import { DataServiceImpl } from '@dxos/echo-pipeline';
-import { getAutomergeObjectCore, getRawDoc, type SpaceDoc, type TypedObject } from '@dxos/echo-schema';
-import { IndexServiceImpl } from '@dxos/indexing';
+import {
+  DataServiceImpl,
+  type ObjectStructure,
+  encodeReference,
+  type SpaceDoc,
+  type LevelDB,
+} from '@dxos/echo-pipeline';
+import { getTypeReference } from '@dxos/echo-schema';
+import { QueryServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -21,18 +29,22 @@ import { TRACE_PROCESSOR, trace as Trace } from '@dxos/tracing';
 import { assignDeep } from '@dxos/util';
 import { WebsocketRpcClient } from '@dxos/websocket-rpc';
 
-import { createDiagnostics } from './diagnostics';
 import { ServiceContext, type ServiceContextRuntimeParams } from './service-context';
 import { ServiceRegistry } from './service-registry';
 import { DevicesServiceImpl } from '../devices';
 import { DevtoolsHostEvents, DevtoolsServiceImpl } from '../devtools';
+import {
+  type CollectDiagnosticsBroadcastHandler,
+  createCollectDiagnosticsBroadcastHandler,
+  createDiagnostics,
+} from '../diagnostics';
 import { IdentityServiceImpl, type CreateIdentityOptions } from '../identity';
 import { InvitationsServiceImpl } from '../invitations';
 import { Lock, type ResourceLock } from '../locks';
 import { LoggingServiceImpl } from '../logging';
 import { NetworkServiceImpl } from '../network';
 import { SpacesServiceImpl } from '../spaces';
-import { createStorageObjects } from '../storage';
+import { createLevel, createStorageObjects } from '../storage';
 import { SystemServiceImpl } from '../system';
 
 export type ClientServicesHostParams = {
@@ -44,6 +56,7 @@ export type ClientServicesHostParams = {
   signalManager?: SignalManager;
   connectionLog?: boolean;
   storage?: Storage;
+  level?: LevelDB;
   lockKey?: string;
   callbacks?: ClientServicesHostCallbacks;
   runtimeParams?: ServiceContextRuntimeParams;
@@ -70,17 +83,20 @@ export class ClientServicesHost {
   private readonly _systemService: SystemServiceImpl;
   private readonly _loggingService: LoggingServiceImpl;
   private readonly _tracingService = TRACE_PROCESSOR.createTraceSender();
+  private _queryService!: QueryServiceImpl;
 
   private _config?: Config;
   private readonly _statusUpdate = new Event<void>();
   private _signalManager?: SignalManager;
   private _networkManager?: NetworkManager;
   private _storage?: Storage;
+  private _level?: Level<string, string>;
   private _callbacks?: ClientServicesHostCallbacks;
   private _devtoolsProxy?: WebsocketRpcClient<{}, ClientServices>;
 
   private _serviceContext!: ServiceContext;
   private readonly _runtimeParams?: ServiceContextRuntimeParams;
+  private diagnosticsBroadcastHandler: CollectDiagnosticsBroadcastHandler;
 
   @Trace.info()
   private _opening = false;
@@ -93,12 +109,14 @@ export class ClientServicesHost {
     transportFactory,
     signalManager,
     storage,
+    level,
     // TODO(wittjosiah): Turn this on by default.
     lockKey,
     callbacks,
     runtimeParams,
   }: ClientServicesHostParams = {}) {
     this._storage = storage;
+    this._level = level;
     this._callbacks = callbacks;
     this._runtimeParams = runtimeParams;
 
@@ -138,6 +156,7 @@ export class ClientServicesHost {
       },
     });
 
+    this.diagnosticsBroadcastHandler = createCollectDiagnosticsBroadcastHandler(this._systemService);
     this._loggingService = new LoggingServiceImpl();
 
     this._serviceRegistry = new ServiceRegistry<ClientServices>(clientServiceBundle, {
@@ -187,6 +206,9 @@ export class ClientServicesHost {
       }
     }
 
+    if (!options.signalManager) {
+      log.warn('running signaling without telemetry metadata.');
+    }
     const {
       connectionLog = true,
       transportFactory = createSimplePeerTransportFactory({
@@ -223,17 +245,30 @@ export class ClientServicesHost {
 
     this._opening = true;
     log('opening...', { lockKey: this._resourceLock?.lockKey });
+
+    if (!this._level) {
+      this._level = await createLevel(this._config.get('runtime.client.storage', {})!);
+    }
+    await this._level.open();
+
     await this._resourceLock?.acquire();
 
     await this._loggingService.open();
 
     this._serviceContext = new ServiceContext(
       this._storage,
+      this._level,
       this._networkManager,
       this._signalManager,
       this._runtimeParams,
     );
 
+    this._queryService = new QueryServiceImpl({
+      indexer: this._serviceContext.indexer,
+      automergeHost: this._serviceContext.automergeHost,
+    });
+    await this._queryService.open(ctx);
+
     this._serviceRegistry.setServices({
       SystemService: this._systemService,
 
@@ -244,11 +279,7 @@ export class ClientServicesHost {
         (profile) => this._serviceContext.broadcastProfileUpdate(profile),
       ),
 
-      InvitationsService: new InvitationsServiceImpl(
-        this._serviceContext.invitations,
-        (invitation) => this._serviceContext.getInvitationHandler(invitation),
-        this._serviceContext.metadataStore,
-      ),
+      InvitationsService: new InvitationsServiceImpl(this._serviceContext.invitationsManager),
 
       DevicesService: new DevicesServiceImpl(this._serviceContext.identityManager),
 
@@ -263,10 +294,7 @@ export class ClientServicesHost {
 
       DataService: new DataServiceImpl(this._serviceContext.automergeHost),
 
-      IndexService: new IndexServiceImpl({
-        indexer: this._serviceContext.indexer,
-        automergeHost: this._serviceContext.automergeHost,
-      }),
+      QueryService: this._queryService,
 
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
@@ -282,11 +310,6 @@ export class ClientServicesHost {
     });
 
     await this._serviceContext.open(ctx);
-    // TODO(nf): move to InvitationManager in ServiceContext?
-    invariant(this.serviceRegistry.services.InvitationsService);
-    const loadedInvitations = await this.serviceRegistry.services.InvitationsService.loadPersistentInvitations();
-
-    log('loaded persistent invitations', { count: loadedInvitations.invitations?.length });
 
     const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
     if (devtoolsProxy) {
@@ -298,6 +321,7 @@ export class ClientServicesHost {
       });
       void this._devtoolsProxy.open();
     }
+    this.diagnosticsBroadcastHandler.start();
 
     this._opening = false;
     this._open = true;
@@ -316,10 +340,13 @@ export class ClientServicesHost {
 
     const deviceKey = this._serviceContext.identityManager.identity?.deviceKey;
     log('closing...', { deviceKey });
+    this.diagnosticsBroadcastHandler.stop();
     await this._devtoolsProxy?.close();
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
+    await this._queryService.close();
     await this._serviceContext.close();
+    await this._level?.close();
     this._open = false;
     this._statusUpdate.emit();
     log('closed', { deviceKey });
@@ -344,18 +371,30 @@ export class ClientServicesHost {
     await this._serviceContext.initialized.wait();
     const space = await this._serviceContext.dataSpaceManager!.createSpace();
 
-    const obj: TypedObject = new Properties(undefined);
-    obj[defaultKey] = identity.identityKey.toHex();
-
     const automergeIndex = space.automergeSpaceState.rootUrl;
     invariant(automergeIndex);
     const document = await this._serviceContext.automergeHost.repo.find<SpaceDoc>(automergeIndex as any);
     await document.whenReady();
 
+    // TODO(dmaretskyi): Better API for low-level data access.
+    const properties: ObjectStructure = {
+      system: {
+        type: encodeReference(getTypeReference(Properties)!),
+      },
+      data: {
+        [defaultKey]: identity.identityKey.toHex(),
+      },
+      meta: {
+        keys: [],
+      },
+    };
+    const propertiesId = PublicKey.random().toHex();
     document.change((doc: SpaceDoc) => {
-      assignDeep(doc, ['objects', getAutomergeObjectCore(obj).id], getRawDoc(obj).handle.docSync());
+      assignDeep(doc, ['objects', propertiesId], properties);
     });
 
+    await this._serviceContext.automergeHost.repo.flush();
+
     return identity;
   }
 }

package/src/packlets/services/service-registry.test.ts

@@ -31,7 +31,7 @@ const serviceBundle = createServiceBundle<TestServices>({
 describe('service registry', () => {
   test('builds a service registry', async () => {
     const remoteSource = 'https://remote.source';
-    const serviceContext = createServiceContext();
+    const serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
 
     const serviceRegistry = new ServiceRegistry(serviceBundle, {

package/src/packlets/services/util.ts

@@ -5,6 +5,8 @@
 import { PublicKey } from '@dxos/keys';
 import { humanize } from '@dxos/util';
 
+export const ClientServicesProviderResource = Symbol.for('dxos.resource.ClientServices');
+
 // TODO(burdon): Move to util.
 
 export type JsonStringifyOptions = {

package/src/packlets/spaces/data-space-manager.test.ts

@@ -20,7 +20,7 @@ describe('DataSpaceManager', () => {
 
     const peer = builder.createPeer();
     await peer.createIdentity();
-    await openAndClose(peer.dataSpaceManager);
+    await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
     const space = await peer.dataSpaceManager.createSpace();
 
@@ -42,7 +42,7 @@ describe('DataSpaceManager', () => {
     const peer2 = builder.createPeer();
     await peer2.createIdentity();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -111,7 +111,7 @@ describe('DataSpaceManager', () => {
     await peer2.createIdentity();
     await peer2.dataSpaceManager.open();
 
-    await openAndClose(peer1.dataSpaceManager, peer2.dataSpaceManager);
+    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -153,7 +153,7 @@ describe('DataSpaceManager', () => {
 
       const peer = builder.createPeer();
       await peer.createIdentity();
-      await openAndClose(peer.dataSpaceManager);
+      await openAndClose(peer.automergeHost, peer.dataSpaceManager);
 
       const space = await peer.dataSpaceManager.createSpace();
       await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);

package/src/packlets/spaces/data-space-manager.ts

@@ -4,7 +4,7 @@
 
 import { Event, synchronized, trackLeaks } from '@dxos/async';
 import { Context, cancelWithContext } from '@dxos/context';
-import { getCredentialAssertion, type CredentialSigner } from '@dxos/credentials';
+import { getCredentialAssertion, type CredentialSigner, type DelegateInvitationCredential } from '@dxos/credentials';
 import { type AutomergeHost, type MetadataStore, type Space, type SpaceManager } from '@dxos/echo-pipeline';
 import { type FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
@@ -12,10 +12,11 @@ import { type Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/protocols';
-import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
+import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
 import { type Credential, type ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { type DelegateSpaceInvitation } from '@dxos/protocols/proto/dxos/halo/invitations';
 import { Gossip, Presence } from '@dxos/teleport-extension-gossip';
 import { type Timeframe } from '@dxos/timeframe';
 import { ComplexMap, deferFunction, forEachAsync } from '@dxos/util';
@@ -23,6 +24,7 @@ import { ComplexMap, deferFunction, forEachAsync } from '@dxos/util';
 import { DataSpace } from './data-space';
 import { spaceGenesis } from './genesis';
 import { createAuthProvider } from '../identity';
+import { type InvitationsManager } from '../invitations';
 
 const PRESENCE_ANNOUNCE_INTERVAL = 10_000;
 const PRESENCE_OFFLINE_TIMEOUT = 20_000;
@@ -78,6 +80,7 @@ export class DataSpaceManager {
     private readonly _signingContext: SigningContext,
     private readonly _feedStore: FeedStore<FeedMessage>,
     private readonly _automergeHost: AutomergeHost,
+    private readonly _invitationsManager: InvitationsManager,
     params?: DataSpaceManagerRuntimeParams,
   ) {
     const {
@@ -247,6 +250,9 @@ export class DataSpaceManager {
         log.warn('auth failure');
       },
       memberKey: this._signingContext.identityKey,
+      onDelegatedInvitationStatusChange: (invitation, isActive) => {
+        return this._handleInvitationStatusChange(dataSpace, invitation, isActive);
+      },
     });
     controlFeed && (await space.setControlFeed(controlFeed));
     dataFeed && (await space.setDataFeed(dataFeed));
@@ -267,6 +273,7 @@ export class DataSpaceManager {
         afterReady: async () => {
           log('after space ready', { space: space.key, open: this._isOpen });
           if (this._isOpen) {
+            await this._createDelegatedInvitations(dataSpace, [...space.spaceState.invitations.entries()]);
             this.updated.emit();
           }
         },
@@ -289,4 +296,43 @@ export class DataSpaceManager {
     this._spaces.set(metadata.key, dataSpace);
     return dataSpace;
   }
+
+  private async _handleInvitationStatusChange(
+    dataSpace: DataSpace | undefined,
+    delegatedInvitation: DelegateInvitationCredential,
+    isActive: boolean,
+  ): Promise<void> {
+    if (dataSpace?.state !== SpaceState.READY) {
+      return;
+    }
+    if (isActive) {
+      await this._createDelegatedInvitations(dataSpace, [
+        [delegatedInvitation.credentialId, delegatedInvitation.invitation],
+      ]);
+    } else {
+      await this._invitationsManager.cancelInvitation(delegatedInvitation.invitation);
+    }
+  }
+
+  private async _createDelegatedInvitations(
+    space: DataSpace,
+    invitations: Array<[PublicKey, DelegateSpaceInvitation]>,
+  ): Promise<void> {
+    const tasks = invitations.map(([credentialId, invitation]) => {
+      return this._invitationsManager.createInvitation({
+        type: Invitation.Type.DELEGATED,
+        kind: Invitation.Kind.SPACE,
+        spaceKey: space.key,
+        authMethod: invitation.authMethod,
+        invitationId: invitation.invitationId,
+        swarmKey: invitation.swarmKey,
+        guestKeypair: invitation.guestKey ? { publicKey: invitation.guestKey } : undefined,
+        lifetime: invitation.expiresOn ? invitation.expiresOn.getTime() - Date.now() : undefined,
+        multiUse: invitation.multiUse,
+        delegationCredentialId: credentialId,
+        persistent: false,
+      });
+    });
+    await Promise.all(tasks);
+  }
 }

package/src/packlets/spaces/data-space.ts

@@ -6,7 +6,15 @@ import { Event, asyncTimeout, scheduleTask, sleep, synchronized, trackLeaks } fr
 import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { cancelWithContext, Context, ContextDisposedError } from '@dxos/context';
 import { timed, warnAfterTimeout } from '@dxos/debug';
-import { type MetadataStore, type Space, createMappedFeedWriter, type AutomergeHost } from '@dxos/echo-pipeline';
+import {
+  type MetadataStore,
+  type Space,
+  createMappedFeedWriter,
+  type AutomergeHost,
+  type SpaceDoc,
+} from '@dxos/echo-pipeline';
+import { AutomergeDocumentLoaderImpl } from '@dxos/echo-pipeline';
+import { TYPE_PROPERTIES } from '@dxos/echo-schema';
 import { type FeedStore } from '@dxos/feed-store';
 import { failedInvariant, invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -26,7 +34,7 @@ import { type GossipMessage } from '@dxos/protocols/proto/dxos/mesh/teleport/gos
 import { type Gossip, type Presence } from '@dxos/teleport-extension-gossip';
 import { Timeframe } from '@dxos/timeframe';
 import { trace } from '@dxos/tracing';
-import { ComplexSet } from '@dxos/util';
+import { ComplexSet, assignDeep } from '@dxos/util';
 
 import { AutomergeSpaceState } from './automerge-space-state';
 import { type SigningContext } from './data-space-manager';
@@ -437,6 +445,47 @@ export class DataSpace {
           };
         }
         break;
+      case CreateEpochRequest.Migration.FRAGMENT_AUTOMERGE_ROOT:
+        {
+          log.info('Fragmenting');
+
+          const currentRootUrl = this._automergeSpaceState.rootUrl;
+          const rootHandle = this._automergeHost.repo.find<SpaceDoc>(currentRootUrl as any);
+          await cancelWithContext(this._ctx, asyncTimeout(rootHandle.whenReady(), 10_000));
+
+          // Find properties object.
+          const objects = Object.entries((rootHandle.docSync() as SpaceDoc).objects!);
+          const properties = objects.find(([_, value]) => value.system.type?.itemId === TYPE_PROPERTIES);
+          const otherObjects = objects.filter(([key]) => key !== properties?.[0]);
+          invariant(properties, 'Properties not found');
+
+          // Create a new space doc with the properties object.
+          const newSpaceDoc: SpaceDoc = { ...rootHandle.docSync(), objects: Object.fromEntries([properties]) };
+          const newRoot = this._automergeHost.repo.create(newSpaceDoc);
+          invariant(typeof newRoot.url === 'string' && newRoot.url.length > 0);
+
+          // Create new automerge documents for all objects.
+          const docLoader = new AutomergeDocumentLoaderImpl(this.key, this._automergeHost.repo);
+          await docLoader.loadSpaceRootDocHandle(this._ctx, { rootUrl: newRoot.url });
+
+          otherObjects.forEach(([key, value]) => {
+            const handle = docLoader.createDocumentForObject(key);
+            handle.change((doc: any) => {
+              assignDeep(doc, ['objects', key], value);
+            });
+          });
+
+          // TODO(mykola): Delete old root.
+
+          // TODO(dmaretskyi): Unify epoch construction.
+          epoch = {
+            previousId: this._automergeSpaceState.lastEpoch?.id,
+            number: (this._automergeSpaceState.lastEpoch?.subject.assertion.number ?? -1) + 1,
+            timeframe: this._automergeSpaceState.lastEpoch?.subject.assertion.timeframe ?? new Timeframe(),
+            automergeRoot: newRoot.url,
+          };
+        }
+        break;
     }
 
     if (!epoch) {

package/src/packlets/spaces/spaces-service.test.ts

@@ -22,7 +22,7 @@ describe('SpacesService', () => {
   let spacesService: SpacesService;
 
   beforeEach(async () => {
-    serviceContext = createServiceContext();
+    serviceContext = await createServiceContext();
     await serviceContext.open(new Context());
     spacesService = new SpacesServiceImpl(serviceContext.identityManager, serviceContext.spaceManager, async () => {
       await serviceContext.initialized.wait();

package/src/packlets/storage/index.ts

@@ -3,3 +3,4 @@
 //
 
 export * from './storage';
+export * from './level';

package/src/packlets/storage/level.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Level } from 'level';
+import path from 'node:path';
+
+import { PublicKey } from '@dxos/keys';
+import { type Runtime } from '@dxos/protocols/proto/dxos/config';
+
+import { getRootPath, isPersistent } from './util';
+
+export const createLevel = async (config: Runtime.Client.Storage) => {
+  const persistent = isPersistent(config);
+  const storagePath = persistent ? path.join(getRootPath(config), 'level') : `/tmp/dxos-${PublicKey.random().toHex()}`;
+  const level = new Level<string, string>(storagePath);
+  await level.open();
+  return level;
+};

package/src/packlets/storage/storage.ts

@@ -4,22 +4,16 @@
 // Copyright 2023 DXOS.org
 //
 
-import { DX_DATA } from '@dxos/client-protocol';
 import { InvalidConfigError } from '@dxos/protocols';
 import { Runtime } from '@dxos/protocols/proto/dxos/config';
 import { createStorage, StorageType } from '@dxos/random-access-storage';
-import { isNode } from '@dxos/util';
 
 import StorageDriver = Runtime.Client.Storage.StorageDriver;
+import { getRootPath } from './util';
 
 // TODO(burdon): Factor out.
 export const createStorageObjects = (config: Runtime.Client.Storage) => {
-  const {
-    persistent = false,
-    keyStore,
-    dataStore,
-    dataRoot = isNode() ? DX_DATA : 'dxos/storage', // TODO(burdon): Factor out const.
-  } = config ?? {};
+  const { persistent = false, keyStore, dataStore } = config ?? {};
 
   if (persistent && dataStore === StorageDriver.RAM) {
     throw new InvalidConfigError('RAM storage cannot be used in persistent mode.');
@@ -37,7 +31,7 @@ export const createStorageObjects = (config: Runtime.Client.Storage) => {
   return {
     storage: createStorage({
       type: persistent ? toStorageType(dataStore) : StorageType.RAM,
-      root: `${dataRoot}/`,
+      root: getRootPath(config),
     }),
   };
 };

package/src/packlets/storage/util.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { DX_DATA } from '@dxos/client-protocol';
+import { Runtime } from '@dxos/protocols/proto/dxos/config';
+import { isNode } from '@dxos/util';
+
+export const getRootPath = (config: Runtime.Client.Storage) => {
+  const { dataRoot = isNode() ? DX_DATA : 'dxos/storage' } = config ?? {};
+  return `${dataRoot}/`;
+};
+
+export const isPersistent = (config: Runtime.Client.Storage) => {
+  const { persistent = false } = config ?? {};
+  return (
+    (config.dataStore !== undefined && config.dataStore !== Runtime.Client.Storage.StorageDriver.RAM) || persistent
+  );
+};

package/src/packlets/system/system-service.ts

@@ -16,7 +16,7 @@ import {
 } from '@dxos/protocols/proto/dxos/client/services';
 import { jsonKeyReplacer, type MaybePromise } from '@dxos/util';
 
-import { type Diagnostics } from '../services';
+import { type Diagnostics } from '../diagnostics';
 import { getPlatform } from '../services/platform';
 
 export type SystemServiceOptions = {