@dxos/client-services 0.5.0 → 0.5.1-main.140276b

package/src/packlets/services/service-context.ts

@@ -8,9 +8,9 @@ import { Trigger } from '@dxos/async';
 import { Context, Resource } from '@dxos/context';
 import { getCredentialAssertion, type CredentialProcessor } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import { AutomergeHost, MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
+import { EchoHost } from '@dxos/echo-db';
+import { MetadataStore, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
-import { IndexMetadataStore, IndexStore, Indexer, createStorageCallbacks } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
@@ -32,7 +32,6 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
-import { createSelectedDocumentsIterator } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -64,9 +63,7 @@ export class ServiceContext extends Resource {
   public readonly identityManager: IdentityManager;
   public readonly invitations: InvitationsHandler;
   public readonly invitationsManager: InvitationsManager;
-  public readonly automergeHost: AutomergeHost;
-  public readonly indexMetadata: IndexMetadataStore;
-  public readonly indexer: Indexer;
+  public readonly echoHost: EchoHost;
 
   // Initialized after identity is initialized.
   public dataSpaceManager?: DataSpaceManager;
@@ -122,19 +119,9 @@ export class ServiceContext extends Resource {
       this._runtimeParams as IdentityManagerRuntimeParams,
     );
 
-    this.indexMetadata = new IndexMetadataStore({ db: level.sublevel('index-metadata') });
-
-    this.automergeHost = new AutomergeHost({
-      directory: storage.createDirectory('automerge'),
-      db: level.sublevel('automerge'),
-      storageCallbacks: createStorageCallbacks({ host: () => this.automergeHost, metadata: this.indexMetadata }),
-    });
-
-    this.indexer = new Indexer({
-      db: this.level,
-      indexStore: new IndexStore({ db: level.sublevel('index-storage') }),
-      metadataStore: this.indexMetadata,
-      loadDocuments: createSelectedDocumentsIterator(this.automergeHost),
+    this.echoHost = new EchoHost({
+      kv: this.level,
+      storage: this.storage,
     });
 
     this.invitations = new InvitationsHandler(this.networkManager);
@@ -166,7 +153,7 @@ export class ServiceContext extends Resource {
     await this.signalManager.open();
     await this.networkManager.open();
 
-    await this.automergeHost.open();
+    await this.echoHost.open(ctx);
     await this.metadataStore.load();
     await this.spaceManager.open();
     await this.identityManager.open(ctx);
@@ -181,20 +168,19 @@ export class ServiceContext extends Resource {
     log('opened');
   }
 
-  protected override async _close() {
+  protected override async _close(ctx: Context) {
     log('closing...');
     if (this._deviceSpaceSync && this.identityManager.identity) {
       await this.identityManager.identity.space.spaceState.removeCredentialProcessor(this._deviceSpaceSync);
     }
-    await this.automergeHost.close();
     await this.dataSpaceManager?.close();
     await this.identityManager.close();
     await this.spaceManager.close();
     await this.feedStore.close();
+    await this.metadataStore.close();
+    await this.echoHost.close(ctx);
     await this.networkManager.close();
     await this.signalManager.close();
-    await this.metadataStore.close();
-    await this.indexer.destroy();
     log('closed');
   }
 
@@ -255,7 +241,7 @@ export class ServiceContext extends Resource {
       this.keyring,
       signingContext,
       this.feedStore,
-      this.automergeHost,
+      this.echoHost,
       this.invitationsManager,
       this._runtimeParams as DataSpaceManagerRuntimeParams,
     );

package/src/packlets/services/service-host.test.ts

@@ -29,6 +29,12 @@ describe('ClientServicesHost', () => {
     isNode() && rmSync(dataRoot, { recursive: true, force: true });
   });
 
+  test('open and close', async () => {
+    const host = createServiceHost(new Config(), new MemorySignalManagerContext());
+    await host.open(new Context());
+    await host.close();
+  });
+
   test('queryCredentials', async () => {
     const host = createServiceHost(new Config(), new MemorySignalManagerContext());
     await host.open(new Context());

package/src/packlets/services/service-host.ts

@@ -8,15 +8,8 @@ import { Event, synchronized } from '@dxos/async';
 import { clientServiceBundle, defaultKey, type ClientServices, Properties } from '@dxos/client-protocol';
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
-import {
-  DataServiceImpl,
-  type ObjectStructure,
-  encodeReference,
-  type SpaceDoc,
-  type LevelDB,
-} from '@dxos/echo-pipeline';
+import { type ObjectStructure, encodeReference, type SpaceDoc, type LevelDB } from '@dxos/echo-pipeline';
 import { getTypeReference } from '@dxos/echo-schema';
-import { QueryServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -83,7 +76,6 @@ export class ClientServicesHost {
   private readonly _systemService: SystemServiceImpl;
   private readonly _loggingService: LoggingServiceImpl;
   private readonly _tracingService = TRACE_PROCESSOR.createTraceSender();
-  private _queryService!: QueryServiceImpl;
 
   private _config?: Config;
   private readonly _statusUpdate = new Event<void>();
@@ -246,13 +238,13 @@ export class ClientServicesHost {
     this._opening = true;
     log('opening...', { lockKey: this._resourceLock?.lockKey });
 
+    await this._resourceLock?.acquire();
+
     if (!this._level) {
       this._level = await createLevel(this._config.get('runtime.client.storage', {})!);
     }
     await this._level.open();
 
-    await this._resourceLock?.acquire();
-
     await this._loggingService.open();
 
     this._serviceContext = new ServiceContext(
@@ -263,12 +255,6 @@ export class ClientServicesHost {
       this._runtimeParams,
     );
 
-    this._queryService = new QueryServiceImpl({
-      indexer: this._serviceContext.indexer,
-      automergeHost: this._serviceContext.automergeHost,
-    });
-    await this._queryService.open(ctx);
-
     this._serviceRegistry.setServices({
       SystemService: this._systemService,
 
@@ -292,9 +278,8 @@ export class ClientServicesHost {
         },
       ),
 
-      DataService: new DataServiceImpl(this._serviceContext.automergeHost),
-
-      QueryService: this._queryService,
+      DataService: this._serviceContext.echoHost.dataService,
+      QueryService: this._serviceContext.echoHost.queryService,
 
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
@@ -344,7 +329,6 @@ export class ClientServicesHost {
     await this._devtoolsProxy?.close();
     this._serviceRegistry.setServices({ SystemService: this._systemService });
     await this._loggingService.close();
-    await this._queryService.close();
     await this._serviceContext.close();
     await this._level?.close();
     this._open = false;
@@ -373,7 +357,7 @@ export class ClientServicesHost {
 
     const automergeIndex = space.automergeSpaceState.rootUrl;
     invariant(automergeIndex);
-    const document = await this._serviceContext.automergeHost.repo.find<SpaceDoc>(automergeIndex as any);
+    const document = await this._serviceContext.echoHost.automergeRepo.find<SpaceDoc>(automergeIndex as any);
     await document.whenReady();
 
     // TODO(dmaretskyi): Better API for low-level data access.
@@ -393,7 +377,7 @@ export class ClientServicesHost {
       assignDeep(doc, ['objects', propertiesId], properties);
     });
 
-    await this._serviceContext.automergeHost.repo.flush();
+    await this._serviceContext.echoHost.flush();
 
     return identity;
   }

package/src/packlets/spaces/data-space-manager.test.ts

@@ -20,7 +20,7 @@ describe('DataSpaceManager', () => {
 
     const peer = builder.createPeer();
     await peer.createIdentity();
-    await openAndClose(peer.automergeHost, peer.dataSpaceManager);
+    await openAndClose(peer.echoHost, peer.dataSpaceManager);
 
     const space = await peer.dataSpaceManager.createSpace();
 
@@ -42,7 +42,7 @@ describe('DataSpaceManager', () => {
     const peer2 = builder.createPeer();
     await peer2.createIdentity();
 
-    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
+    await openAndClose(peer1.echoHost, peer1.dataSpaceManager, peer2.echoHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -111,7 +111,7 @@ describe('DataSpaceManager', () => {
     await peer2.createIdentity();
     await peer2.dataSpaceManager.open();
 
-    await openAndClose(peer1.automergeHost, peer1.dataSpaceManager, peer2.automergeHost, peer2.dataSpaceManager);
+    await openAndClose(peer1.echoHost, peer1.dataSpaceManager, peer2.echoHost, peer2.dataSpaceManager);
 
     const space1 = await peer1.dataSpaceManager.createSpace();
     await space1.inner.controlPipeline.state.waitUntilTimeframe(space1.inner.controlPipeline.state.endTimeframe);
@@ -153,7 +153,7 @@ describe('DataSpaceManager', () => {
 
       const peer = builder.createPeer();
       await peer.createIdentity();
-      await openAndClose(peer.automergeHost, peer.dataSpaceManager);
+      await openAndClose(peer.echoHost, peer.dataSpaceManager);
 
       const space = await peer.dataSpaceManager.createSpace();
       await space.inner.controlPipeline.state.waitUntilTimeframe(space.inner.controlPipeline.state.endTimeframe);

package/src/packlets/spaces/data-space-manager.ts

@@ -5,7 +5,8 @@
 import { Event, synchronized, trackLeaks } from '@dxos/async';
 import { Context, cancelWithContext } from '@dxos/context';
 import { getCredentialAssertion, type CredentialSigner, type DelegateInvitationCredential } from '@dxos/credentials';
-import { type AutomergeHost, type MetadataStore, type Space, type SpaceManager } from '@dxos/echo-pipeline';
+import { type EchoHost } from '@dxos/echo-db';
+import { type MetadataStore, type Space, type SpaceManager } from '@dxos/echo-pipeline';
 import { type FeedStore } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
@@ -79,7 +80,7 @@ export class DataSpaceManager {
     private readonly _keyring: Keyring,
     private readonly _signingContext: SigningContext,
     private readonly _feedStore: FeedStore<FeedMessage>,
-    private readonly _automergeHost: AutomergeHost,
+    private readonly _echoHost: EchoHost,
     private readonly _invitationsManager: InvitationsManager,
     params?: DataSpaceManagerRuntimeParams,
   ) {
@@ -152,14 +153,10 @@ export class DataSpaceManager {
 
     log('creating space...', { spaceKey });
 
-    const automergeRoot = this._automergeHost.repo.create();
-    automergeRoot.change((doc: any) => {
-      doc.access = { spaceKey: spaceKey.toHex() };
-    });
-
+    const automergeRootUrl = await this._echoHost.createSpaceRoot(spaceKey);
     const space = await this._constructSpace(metadata);
 
-    const credentials = await spaceGenesis(this._keyring, this._signingContext, space.inner, automergeRoot.url);
+    const credentials = await spaceGenesis(this._keyring, this._signingContext, space.inner, automergeRootUrl);
     await this._metadataStore.addSpace(metadata);
 
     const memberCredential = credentials[1];
@@ -243,8 +240,8 @@ export class DataSpaceManager {
           gossip.createExtension({ remotePeerId: session.remotePeerId }),
         );
         session.addExtension('dxos.mesh.teleport.notarization', dataSpace.notarizationPlugin.createExtension());
-        this._automergeHost.authorizeDevice(space.key, session.remotePeerId);
-        session.addExtension('dxos.mesh.teleport.automerge', this._automergeHost.createExtension());
+        this._echoHost.authorizeDevice(space.key, session.remotePeerId);
+        session.addExtension('dxos.mesh.teleport.automerge', this._echoHost.createReplicationExtension());
       },
       onAuthFailure: () => {
         log.warn('auth failure');
@@ -265,6 +262,7 @@ export class DataSpaceManager {
       presence,
       keyring: this._keyring,
       feedStore: this._feedStore,
+      echoHost: this._echoHost,
       signingContext: this._signingContext,
       callbacks: {
         beforeReady: async () => {
@@ -282,7 +280,6 @@ export class DataSpaceManager {
         },
       },
       cache: metadata.cache,
-      automergeHost: this._automergeHost,
     });
 
     if (metadata.state !== SpaceState.INACTIVE) {

package/src/packlets/spaces/data-space.ts

@@ -6,13 +6,8 @@ import { Event, asyncTimeout, scheduleTask, sleep, synchronized, trackLeaks } fr
 import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { cancelWithContext, Context, ContextDisposedError } from '@dxos/context';
 import { timed, warnAfterTimeout } from '@dxos/debug';
-import {
-  type MetadataStore,
-  type Space,
-  createMappedFeedWriter,
-  type AutomergeHost,
-  type SpaceDoc,
-} from '@dxos/echo-pipeline';
+import { type EchoHost } from '@dxos/echo-db';
+import { type MetadataStore, type Space, createMappedFeedWriter, type SpaceDoc } from '@dxos/echo-pipeline';
 import { AutomergeDocumentLoaderImpl } from '@dxos/echo-pipeline';
 import { TYPE_PROPERTIES } from '@dxos/echo-schema';
 import { type FeedStore } from '@dxos/feed-store';
@@ -66,10 +61,10 @@ export type DataSpaceParams = {
   presence: Presence;
   keyring: Keyring;
   feedStore: FeedStore<FeedMessage>;
+  echoHost: EchoHost;
   signingContext: SigningContext;
   callbacks?: DataSpaceCallbacks;
   cache?: SpaceCache;
-  automergeHost: AutomergeHost;
 };
 
 export type CreateEpochOptions = {
@@ -92,7 +87,7 @@ export class DataSpace {
   private readonly _notarizationPlugin = new NotarizationPlugin();
   private readonly _callbacks: DataSpaceCallbacks;
   private readonly _cache?: SpaceCache = undefined;
-  private readonly _automergeHost: AutomergeHost;
+  private readonly _echoHost: EchoHost;
 
   // TODO(dmaretskyi): Move into Space?
   private readonly _automergeSpaceState = new AutomergeSpaceState((rootUrl) => this._onNewAutomergeRoot(rootUrl));
@@ -120,7 +115,7 @@ export class DataSpace {
     this._metadataStore = params.metadataStore;
     this._signingContext = params.signingContext;
     this._callbacks = params.callbacks ?? {};
-    this._automergeHost = params.automergeHost;
+    this._echoHost = params.echoHost;
 
     this.authVerifier = new TrustedKeySetAuthVerifier({
       trustedKeysProvider: () =>
@@ -363,8 +358,8 @@ export class DataSpace {
 
   private _onNewAutomergeRoot(rootUrl: string) {
     log('loading automerge root doc for space', { space: this.key, rootUrl });
-    this._automergeHost._requestedDocs.add(rootUrl as any);
-    const handle = this._automergeHost.repo.find(rootUrl as any);
+    this._echoHost.replicateDocument(rootUrl);
+    const handle = this._echoHost.automergeRepo.find(rootUrl as any);
 
     queueMicrotask(async () => {
       try {
@@ -419,7 +414,7 @@ export class DataSpace {
         break;
       case CreateEpochRequest.Migration.INIT_AUTOMERGE:
         {
-          const document = this._automergeHost.repo.create();
+          const document = this._echoHost.automergeRepo.create();
           // TODO(dmaretskyi): Unify epoch construction.
           epoch = {
             previousId: this._automergeSpaceState.lastEpoch?.id,
@@ -432,9 +427,9 @@ export class DataSpace {
       case CreateEpochRequest.Migration.PRUNE_AUTOMERGE_ROOT_HISTORY:
         {
           const currentRootUrl = this._automergeSpaceState.rootUrl;
-          const rootHandle = this._automergeHost.repo.find(currentRootUrl as any);
+          const rootHandle = this._echoHost.automergeRepo.find(currentRootUrl as any);
           await cancelWithContext(this._ctx, asyncTimeout(rootHandle.whenReady(), 10_000));
-          const newRoot = this._automergeHost.repo.create(rootHandle.docSync());
+          const newRoot = this._echoHost.automergeRepo.create(rootHandle.docSync());
           invariant(typeof newRoot.url === 'string' && newRoot.url.length > 0);
           // TODO(dmaretskyi): Unify epoch construction.
           epoch = {
@@ -450,7 +445,7 @@ export class DataSpace {
           log.info('Fragmenting');
 
           const currentRootUrl = this._automergeSpaceState.rootUrl;
-          const rootHandle = this._automergeHost.repo.find<SpaceDoc>(currentRootUrl as any);
+          const rootHandle = this._echoHost.automergeRepo.find<SpaceDoc>(currentRootUrl as any);
           await cancelWithContext(this._ctx, asyncTimeout(rootHandle.whenReady(), 10_000));
 
           // Find properties object.
@@ -461,11 +456,11 @@ export class DataSpace {
 
           // Create a new space doc with the properties object.
           const newSpaceDoc: SpaceDoc = { ...rootHandle.docSync(), objects: Object.fromEntries([properties]) };
-          const newRoot = this._automergeHost.repo.create(newSpaceDoc);
+          const newRoot = this._echoHost.automergeRepo.create(newSpaceDoc);
           invariant(typeof newRoot.url === 'string' && newRoot.url.length > 0);
 
           // Create new automerge documents for all objects.
-          const docLoader = new AutomergeDocumentLoaderImpl(this.key, this._automergeHost.repo);
+          const docLoader = new AutomergeDocumentLoaderImpl(this.key, this._echoHost.automergeRepo);
           await docLoader.loadSpaceRootDocHandle(this._ctx, { rootUrl: newRoot.url });
 
           otherObjects.forEach(([key, value]) => {

package/src/packlets/storage/level.ts

@@ -14,6 +14,7 @@ export const createLevel = async (config: Runtime.Client.Storage) => {
   const persistent = isPersistent(config);
   const storagePath = persistent ? path.join(getRootPath(config), 'level') : `/tmp/dxos-${PublicKey.random().toHex()}`;
   const level = new Level<string, string>(storagePath);
+  // TODO(dmaretskyi): This function shouldn't call open - .
   await level.open();
   return level;
 };

package/src/packlets/testing/test-builder.ts

@@ -6,14 +6,8 @@ import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
 import { createCredentialSignerWithChain, CredentialGenerator } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
-import {
-  AutomergeHost,
-  MetadataStore,
-  type LevelDB,
-  SnapshotStore,
-  SpaceManager,
-  valueEncoding,
-} from '@dxos/echo-pipeline';
+import { EchoHost } from '@dxos/echo-db';
+import { MetadataStore, type LevelDB, SnapshotStore, SpaceManager, valueEncoding } from '@dxos/echo-pipeline';
 import { createTestLevel } from '@dxos/echo-pipeline/testing';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
 import { Keyring } from '@dxos/keyring';
@@ -105,7 +99,7 @@ export type TestPeerProps = {
   snapshotStore?: SnapshotStore;
   signingContext?: SigningContext;
   blobStore?: BlobStore;
-  automergeHost?: AutomergeHost;
+  echoHost?: EchoHost;
   invitationsManager?: InvitationsManager;
 };
 
@@ -178,9 +172,10 @@ export class TestPeer {
     return this._props.signingContext ?? failUndefined();
   }
 
-  get automergeHost() {
-    return (this._props.automergeHost ??= new AutomergeHost({
-      db: this.level.sublevel('automerge'),
+  get echoHost() {
+    return (this._props.echoHost ??= new EchoHost({
+      kv: this.level,
+      storage: this.storage,
     }));
   }
 
@@ -191,7 +186,7 @@ export class TestPeer {
       this.keyring,
       this.identity,
       this.feedStore,
-      this.automergeHost,
+      this.echoHost,
       this.invitationsManager,
     ));
   }

package/src/packlets/vault/worker-runtime.ts

@@ -120,7 +120,9 @@ export class WorkerRuntime {
       this._sessions.delete(session);
       if (this._sessions.size === 0) {
         // Terminate the worker when all sessions are closed.
-        self.close();
+        if (globalThis.self) {
+          self.close();
+        }
       } else {
         this._reconnectWebrtc();
       }

package/src/version.ts

@@ -1,5 +1 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-export const DXOS_VERSION = '0.5.0'; // {x-release-please-version}
+export const DXOS_VERSION = "0.5.1-main.140276b";