@dxos/client-services 0.4.7-next.f4b92be → 0.4.8-main.00e6f19

package/src/packlets/logging/logging-service.ts

@@ -4,6 +4,7 @@
 
 import { Event } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
+import { PublicKey } from '@dxos/keys';
 import {
   type LogLevel,
   type LogProcessor,
@@ -21,14 +22,15 @@ import {
   type QueryMetricsRequest,
   type QueryMetricsResponse,
 } from '@dxos/protocols/proto/dxos/client/services';
-import { jsonify, numericalValues, tracer } from '@dxos/util';
+import { getDebugName, jsonify, numericalValues, tracer } from '@dxos/util';
 
 /**
  * Logging service used to spy on logs of the host.
  */
 export class LoggingServiceImpl implements LoggingService {
   private readonly _logs = new Event<NaturalLogEntry>();
-  private readonly _started = new Date();
+  private readonly _started = Date.now();
+  private readonly _sessionId = PublicKey.random().toHex();
 
   async open() {
     log.runtimeConfig.processors.push(this._logProcessor);
@@ -116,6 +118,11 @@ export class LoggingServiceImpl implements LoggingService {
             // TODO(dmaretskyi): Fix proto.
             file: entry.meta?.F ?? '',
             line: entry.meta?.L ?? 0,
+            scope: {
+              hostSessionId: this._sessionId,
+              uptimeSeconds: (Date.now() - this._started) / 1000,
+              name: getDebugName(entry.meta?.S),
+            },
           },
         };
 

package/src/packlets/services/service-context.ts

@@ -14,8 +14,8 @@ import {
   SnapshotStore,
   AutomergeHost,
 } from '@dxos/echo-pipeline';
-import { IndexMetadataStore } from '@dxos/echo-schema';
 import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { IndexMetadataStore, IndexStore, Indexer } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
@@ -38,6 +38,7 @@ import {
   type IdentityManagerRuntimeParams,
   type JoinIdentityParams,
 } from '../identity';
+import { createGetAllDocuments, createLoadDocuments } from '../indexing';
 import {
   DeviceInvitationProtocol,
   InvitationsHandler,
@@ -70,6 +71,7 @@ export class ServiceContext {
   public readonly invitations: InvitationsHandler;
   public readonly automergeHost: AutomergeHost;
   public readonly indexMetadata: IndexMetadataStore;
+  public readonly indexer: Indexer;
 
   // Initialized after identity is initialized.
   public dataSpaceManager?: DataSpaceManager;
@@ -92,7 +94,6 @@ export class ServiceContext {
   ) {
     // TODO(burdon): Move strings to constants.
     this.metadataStore = new MetadataStore(storage.createDirectory('metadata'));
-    this.indexMetadata = new IndexMetadataStore({ directory: storage.createDirectory('index-metadata') });
     this.snapshotStore = new SnapshotStore(storage.createDirectory('snapshots'));
     this.blobStore = new BlobStore(storage.createDirectory('blobs'));
 
@@ -125,11 +126,20 @@ export class ServiceContext {
       this._runtimeParams as IdentityManagerRuntimeParams,
     );
 
+    this.indexMetadata = new IndexMetadataStore({ directory: storage.createDirectory('index-metadata') });
+
     this.automergeHost = new AutomergeHost({
       directory: storage.createDirectory('automerge'),
       metadata: this.indexMetadata,
     });
 
+    this.indexer = new Indexer({
+      indexStore: new IndexStore({ directory: storage.createDirectory('index-store') }),
+      metadataStore: this.indexMetadata,
+      loadDocuments: createLoadDocuments(this.automergeHost),
+      getAllDocuments: createGetAllDocuments(this.automergeHost),
+    });
+
     this.invitations = new InvitationsHandler(this.networkManager);
 
     // TODO(burdon): _initialize called in multiple places.
@@ -178,6 +188,7 @@ export class ServiceContext {
     await this.signalManager.close();
     this.dataServiceSubscriptions.clear();
     await this.metadataStore.close();
+    await this.indexer.destroy();
     log('closed');
   }
 

package/src/packlets/services/service-host.ts

@@ -9,6 +9,7 @@ import { Context } from '@dxos/context';
 import { DocumentModel } from '@dxos/document-model';
 import { DataServiceImpl } from '@dxos/echo-pipeline';
 import { type TypedObject, getRawDoc, type SpaceDoc, getAutomergeObjectCore } from '@dxos/echo-schema';
+import { IndexServiceImpl } from '@dxos/indexing';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -256,8 +257,10 @@ export class ClientServicesHost {
         (profile) => this._serviceContext.broadcastProfileUpdate(profile),
       ),
 
-      InvitationsService: new InvitationsServiceImpl(this._serviceContext.invitations, (invitation) =>
-        this._serviceContext.getInvitationHandler(invitation),
+      InvitationsService: new InvitationsServiceImpl(
+        this._serviceContext.invitations,
+        (invitation) => this._serviceContext.getInvitationHandler(invitation),
+        this._serviceContext.metadataStore,
       ),
 
       DevicesService: new DevicesServiceImpl(this._serviceContext.identityManager),
@@ -277,6 +280,11 @@ export class ClientServicesHost {
         this._serviceContext.automergeHost,
       ),
 
+      IndexService: new IndexServiceImpl({
+        indexer: this._serviceContext.indexer,
+        automergeHost: this._serviceContext.automergeHost,
+      }),
+
       NetworkService: new NetworkServiceImpl(this._serviceContext.networkManager, this._serviceContext.signalManager),
 
       LoggingService: this._loggingService,
@@ -291,6 +299,11 @@ export class ClientServicesHost {
     });
 
     await this._serviceContext.open(ctx);
+    // TODO(nf): move to InvitationManager in ServiceContext?
+    invariant(this.serviceRegistry.services.InvitationsService);
+    const loadedInvitations = await this.serviceRegistry.services.InvitationsService.loadPersistentInvitations();
+
+    log('loaded persistent invitations', { count: loadedInvitations.invitations?.length });
 
     const devtoolsProxy = this._config?.get('runtime.client.devtoolsProxy');
     if (devtoolsProxy) {

package/src/packlets/spaces/automerge-space-state.ts

@@ -9,6 +9,8 @@ export class AutomergeSpaceState implements CredentialProcessor {
   public rootUrl: string | undefined = undefined;
   public lastEpoch: SpecificCredential<Epoch> | undefined = undefined;
 
+  private _isProcessingRootDocs = false;
+
   constructor(private readonly _onNewRoot: (rootUrl: string) => void) {}
 
   async processCredential(credential: Credential) {
@@ -20,7 +22,20 @@ export class AutomergeSpaceState implements CredentialProcessor {
     if (credential.subject.assertion.automergeRoot) {
       this.rootUrl = credential.subject.assertion.automergeRoot;
 
+      if (this._isProcessingRootDocs) {
+        this._onNewRoot(this.rootUrl);
+      }
+    }
+  }
+
+  startProcessingRootDocs() {
+    if (this._isProcessingRootDocs) {
+      return;
+    }
+
+    if (this.rootUrl) {
       this._onNewRoot(this.rootUrl);
     }
+    this._isProcessingRootDocs = true;
   }
 }

package/src/packlets/spaces/data-space.ts

@@ -2,7 +2,7 @@
 // Copyright 2022 DXOS.org
 //
 
-import { Event, scheduleTask, sleep, synchronized, trackLeaks } from '@dxos/async';
+import { Event, asyncTimeout, scheduleTask, sleep, synchronized, trackLeaks } from '@dxos/async';
 import { AUTH_TIMEOUT } from '@dxos/client-protocol';
 import { cancelWithContext, Context, ContextDisposedError } from '@dxos/context';
 import { timed, warnAfterTimeout } from '@dxos/debug';
@@ -15,7 +15,7 @@ import {
   type AutomergeHost,
 } from '@dxos/echo-pipeline';
 import { type FeedStore } from '@dxos/feed-store';
-import { failedInvariant } from '@dxos/invariant';
+import { failedInvariant, invariant } from '@dxos/invariant';
 import { type Keyring } from '@dxos/keyring';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -276,6 +276,8 @@ export class DataSpace {
     // Allow other tasks to run before loading the data pipeline.
     await sleep(1);
 
+    this._automergeSpaceState.startProcessingRootDocs();
+
     await this._inner.initializeDataPipeline();
 
     this.metrics.dataPipelineOpen = new Date();
@@ -423,18 +425,38 @@ export class DataSpace {
       case undefined:
       case CreateEpochRequest.Migration.NONE:
         {
+          // TODO(dmaretskyi): Unify epoch construction.
           epoch = await this.dataPipeline.createEpoch();
         }
         break;
-      case CreateEpochRequest.Migration.INIT_AUTOMERGE: {
-        const document = this._automergeHost.repo.create();
-        epoch = {
-          previousId: this._automergeSpaceState.lastEpoch?.id,
-          number: (this._automergeSpaceState.lastEpoch?.subject.assertion.number ?? -1) + 1,
-          timeframe: this._automergeSpaceState.lastEpoch?.subject.assertion.timeframe ?? new Timeframe(),
-          automergeRoot: document.url,
-        };
-      }
+      case CreateEpochRequest.Migration.INIT_AUTOMERGE:
+        {
+          const document = this._automergeHost.repo.create();
+          // TODO(dmaretskyi): Unify epoch construction.
+          epoch = {
+            previousId: this._automergeSpaceState.lastEpoch?.id,
+            number: (this._automergeSpaceState.lastEpoch?.subject.assertion.number ?? -1) + 1,
+            timeframe: this._automergeSpaceState.lastEpoch?.subject.assertion.timeframe ?? new Timeframe(),
+            automergeRoot: document.url,
+          };
+        }
+        break;
+      case CreateEpochRequest.Migration.PRUNE_AUTOMERGE_ROOT_HISTORY:
+        {
+          const currentRootUrl = this._automergeSpaceState.rootUrl;
+          const rootHandle = this._automergeHost.repo.find(currentRootUrl as any);
+          await cancelWithContext(this._ctx, asyncTimeout(rootHandle.whenReady(), 10_000));
+          const newRoot = this._automergeHost.repo.create(rootHandle.docSync());
+          invariant(typeof newRoot.url === 'string' && newRoot.url.length > 0);
+          // TODO(dmaretskyi): Unify epoch construction.
+          epoch = {
+            previousId: this._automergeSpaceState.lastEpoch?.id,
+            number: (this._automergeSpaceState.lastEpoch?.subject.assertion.number ?? -1) + 1,
+            timeframe: this._automergeSpaceState.lastEpoch?.subject.assertion.timeframe ?? new Timeframe(),
+            automergeRoot: newRoot.url,
+          };
+        }
+        break;
     }
 
     if (!epoch) {

package/src/packlets/spaces/spaces-service.ts

@@ -178,10 +178,10 @@ export class SpacesServiceImpl implements SpacesService {
     }
   }
 
-  async createEpoch({ spaceKey }: CreateEpochRequest) {
+  async createEpoch({ spaceKey, migration }: CreateEpochRequest) {
     const dataSpaceManager = await this._getDataSpaceManager();
     const space = dataSpaceManager.spaces.get(spaceKey) ?? raise(new SpaceNotFoundError(spaceKey));
-    await space.createEpoch();
+    await space.createEpoch({ migration });
   }
 
   private _serializeSpace(space: DataSpace): Space {

package/src/version.ts

@@ -1 +1 @@
-export const DXOS_VERSION = "0.4.7-next.f4b92be";
+export const DXOS_VERSION = "0.4.8-main.00e6f19";