@dxos/client-services 0.8.4-main.a4bbb77 → 0.8.4-main.abd8ff62ef

package/src/packlets/spaces/spaces-service.ts

@@ -5,7 +5,9 @@
 import type { AutomergeUrl } from '@automerge/automerge-repo';
 
 import { SubscriptionList, UpdateScheduler, scheduleTask } from '@dxos/async';
+import { type RequestOptions } from '@dxos/codec-protobuf';
 import { Stream } from '@dxos/codec-protobuf/stream';
+import { Context } from '@dxos/context';
 import {
   type CredentialProcessor,
   createAdmissionCredentials,
@@ -13,7 +15,8 @@ import {
   getCredentialAssertion,
 } from '@dxos/credentials';
 import { raise } from '@dxos/debug';
-import { type SpaceManager } from '@dxos/echo-pipeline';
+import { type EchoHost, type SpaceManager } from '@dxos/echo-pipeline';
+import { type DatabaseDirectory } from '@dxos/echo-protocol';
 import { writeMessages } from '@dxos/feed-store';
 import { assertArgument, assertState, invariant } from '@dxos/invariant';
 import { SpaceId } from '@dxos/keys';
@@ -21,6 +24,7 @@ import { log } from '@dxos/log';
 import {
   ApiError,
   AuthorizationError,
+  FeedProtocol,
   IdentityNotInitializedError,
   SpaceNotFoundError,
   encodeError,
@@ -40,11 +44,13 @@ import {
   type QueryCredentialsRequest,
   type QuerySpacesResponse,
   type Space,
+  SpaceArchive,
   SpaceMember,
   SpaceState,
   type SpacesService,
   type SubscribeMessagesRequest,
   type UpdateMemberRoleRequest,
+  type CreateSpaceRequest,
   type UpdateSpaceRequest,
   type WriteCredentialsRequest,
 } from '@dxos/protocols/proto/dxos/client/services';
@@ -54,8 +60,14 @@ import { trace } from '@dxos/tracing';
 import { type Provider } from '@dxos/util';
 
 import { type IdentityManager } from '../identity';
-import { SpaceArchiveWriter, extractSpaceArchive } from '../space-export';
-
+import {
+  SpaceArchiveWriter,
+  detectSpaceArchiveFormat,
+  extractSpaceArchive,
+  readSerializedSpaceArchive,
+  writeSerializedSpaceArchive,
+  objJsonToObjectStructure,
+} from '../space-export';
 import { type DataSpace } from './data-space';
 import { type DataSpaceManager } from './data-space-manager';
 
@@ -63,37 +75,43 @@ export class SpacesServiceImpl implements SpacesService {
   constructor(
     private readonly _identityManager: IdentityManager,
     private readonly _spaceManager: SpaceManager,
+    private readonly _echoHost: EchoHost,
     private readonly _getDataSpaceManager: Provider<Promise<DataSpaceManager>>,
   ) {}
 
-  async createSpace(): Promise<Space> {
+  async createSpace(request: CreateSpaceRequest, options?: RequestOptions): Promise<Space> {
     this._requireIdentity();
+    const ctx = options?.ctx ?? new Context();
     const dataSpaceManager = await this._getDataSpaceManager();
-    const space = await dataSpaceManager.createSpace();
+    const space = await dataSpaceManager.createSpace(ctx, {
+      tags: request?.tags,
+      membershipPolicy: request?.membershipPolicy,
+    });
     await this._updateMetrics();
     return this._serializeSpace(space);
   }
 
-  async updateSpace({ spaceKey, state, edgeReplication }: UpdateSpaceRequest): Promise<void> {
+  async updateSpace({ spaceKey, state, edgeReplication }: UpdateSpaceRequest, options?: RequestOptions): Promise<void> {
+    const ctx = options?.ctx ?? Context.default();
     const dataSpaceManager = await this._getDataSpaceManager();
     const space = dataSpaceManager.spaces.get(spaceKey) ?? raise(new SpaceNotFoundError(spaceKey));
 
     if (state) {
       switch (state) {
         case SpaceState.SPACE_ACTIVE:
-          await space.activate();
+          await space.activate(ctx);
           break;
 
         case SpaceState.SPACE_INACTIVE:
-          await space.deactivate();
+          await space.deactivate(ctx);
           break;
         default:
-          throw new ApiError('Invalid space state');
+          throw new ApiError({ message: 'Invalid space state' });
       }
     }
 
     if (edgeReplication !== undefined) {
-      await dataSpaceManager.setSpaceEdgeReplicationSetting(spaceKey, edgeReplication);
+      await dataSpaceManager.setSpaceEdgeReplicationSetting(ctx, spaceKey, edgeReplication);
     }
   }
 
@@ -104,9 +122,12 @@ export class SpacesServiceImpl implements SpacesService {
       throw new SpaceNotFoundError(request.spaceKey);
     }
     if (!space.spaceState.hasMembershipManagementPermission(identity.identityKey)) {
-      throw new AuthorizationError('No member management permission.', {
-        spaceKey: space.key,
-        role: space.spaceState.getMemberRole(identity.identityKey),
+      throw new AuthorizationError({
+        message: 'No member management permission.',
+        context: {
+          spaceKey: space.key,
+          role: space.spaceState.getMemberRole(identity.identityKey),
+        },
       });
     }
     const credentials = await createAdmissionCredentials(
@@ -259,18 +280,26 @@ export class SpacesServiceImpl implements SpacesService {
     });
   }
 
-  async joinBySpaceKey({ spaceKey }: JoinBySpaceKeyRequest): Promise<JoinSpaceResponse> {
+  async joinBySpaceKey({ spaceKey }: JoinBySpaceKeyRequest, options?: RequestOptions): Promise<JoinSpaceResponse> {
+    const ctx = options?.ctx ?? Context.default();
     const dataSpaceManager = await this._getDataSpaceManager();
-    const credential = await dataSpaceManager.requestSpaceAdmissionCredential(spaceKey);
-    return this._joinByAdmission({ credential });
+    const credential = await dataSpaceManager.requestSpaceAdmissionCredential(ctx, spaceKey);
+    return this._joinByAdmission(ctx, { credential });
   }
 
   async exportSpace(request: ExportSpaceRequest): Promise<ExportSpaceResponse> {
-    await using writer = await new SpaceArchiveWriter().open();
     assertArgument(SpaceId.isValid(request.spaceId), 'spaceId', 'Invalid space ID');
 
     const dataSpaceManager = await this._getDataSpaceManager();
     const space = dataSpaceManager.getSpaceById(request.spaceId) ?? raise(new Error('Space not found'));
+
+    const format = request.format ?? SpaceArchive.Format.BINARY;
+    if (format === SpaceArchive.Format.JSON) {
+      const archive = await writeSerializedSpaceArchive({ space, echoHost: this._echoHost });
+      return { archive };
+    }
+
+    await using writer = await new SpaceArchiveWriter().open();
     await writer.begin({ spaceId: space.id });
     const rootUrl = space.automergeSpaceState.lastEpoch?.subject.assertion.automergeRoot;
     assertState(rootUrl, 'Space does not have a root URL');
@@ -280,15 +309,40 @@ export class SpacesServiceImpl implements SpacesService {
       await writer.writeDocument(documentId, data);
     }
 
+    const feeds = await space.getAllFeeds();
+    for (const feed of feeds) {
+      const archiveBlocks = feed.blocks.map((block) => ({
+        actorId: block.actorId,
+        sequence: block.sequence,
+        prevActorId: block.prevActorId,
+        prevSequence: block.prevSequence,
+        position: block.position,
+        timestamp: block.timestamp,
+        data: Buffer.from(block.data).toString('base64'),
+      }));
+      await writer.writeFeed(feed.feedId, feed.feedNamespace, archiveBlocks);
+    }
+
     const archive = await writer.finish();
     return { archive };
   }
 
-  async importSpace(request: ImportSpaceRequest): Promise<ImportSpaceResponse> {
+  async importSpace(request: ImportSpaceRequest, options?: RequestOptions): Promise<ImportSpaceResponse> {
+    const ctx = options?.ctx ?? Context.default();
     const dataSpaceManager = await this._getDataSpaceManager();
+
+    const format = request.archive.format ?? detectSpaceArchiveFormat(request.archive);
+    if (format === SpaceArchive.Format.JSON) {
+      const serialized = readSerializedSpaceArchive(request.archive);
+      const space = await dataSpaceManager.createSpace(ctx);
+      await this._hydrateSpaceFromSerialized(space, serialized);
+      await this._updateMetrics();
+      return { newSpaceId: space.id };
+    }
+
     const extracted = await extractSpaceArchive(request.archive);
     invariant(extracted.metadata.echo?.currentRootUrl, 'Space archive does not contain a root URL');
-    const space = await dataSpaceManager.createSpace({
+    const space = await dataSpaceManager.createSpace(ctx, {
       documents: extracted.documents,
       rootUrl: extracted.metadata.echo?.currentRootUrl as AutomergeUrl,
     });
@@ -296,7 +350,49 @@ export class SpacesServiceImpl implements SpacesService {
     return { newSpaceId: space.id };
   }
 
-  private async _joinByAdmission({ credential }: ContactAdmission): Promise<JoinSpaceResponse> {
+  /**
+   * Populate a freshly-created space with the objects and feed messages described in a {@link SerializedSpace}.
+   *
+   * Objects are written directly into the space's automerge root document as inline
+   * {@link ObjectStructure} entries; feed messages are appended to the appropriate queue
+   * via {@link EchoHost.queuesService}.
+   */
+  private async _hydrateSpaceFromSerialized(
+    space: DataSpace,
+    serialized: ReturnType<typeof readSerializedSpaceArchive>,
+  ): Promise<void> {
+    const databaseRoot = space.databaseRoot;
+    assertState(databaseRoot, 'Space database root is not ready');
+
+    databaseRoot.handle.change((doc: DatabaseDirectory) => {
+      if (!doc.objects) {
+        doc.objects = {};
+      }
+      for (const obj of serialized.objects) {
+        doc.objects[obj.id] = objJsonToObjectStructure(obj);
+      }
+    });
+
+    for (const feed of serialized.feeds ?? []) {
+      if (feed.messages.length === 0) {
+        continue;
+      }
+      const namespace =
+        feed.namespace === 'trace' ? FeedProtocol.WellKnownNamespaces.trace : FeedProtocol.WellKnownNamespaces.data;
+      try {
+        await this._echoHost.queuesService.insertIntoQueue({
+          spaceId: space.id,
+          queueId: feed.feedObjectId,
+          subspaceTag: namespace,
+          objects: feed.messages.map((message) => JSON.stringify(message)),
+        });
+      } catch (err) {
+        log.warn('failed to import feed data', { feedObjectId: feed.feedObjectId, error: err });
+      }
+    }
+  }
+
+  private async _joinByAdmission(ctx: Context, { credential }: ContactAdmission): Promise<JoinSpaceResponse> {
     const assertion = getCredentialAssertion(credential);
     invariant(assertion['@type'] === 'dxos.halo.credentials.SpaceMember', 'Invalid credential');
     const myIdentity = this._identityManager.identity;
@@ -305,9 +401,10 @@ export class SpacesServiceImpl implements SpacesService {
     const dataSpaceManager = await this._getDataSpaceManager();
     let dataSpace = dataSpaceManager.spaces.get(assertion.spaceKey);
     if (!dataSpace) {
-      dataSpace = await dataSpaceManager.acceptSpace({
+      dataSpace = await dataSpaceManager.acceptSpace(ctx, {
         spaceKey: assertion.spaceKey,
         genesisFeedKey: assertion.genesisFeedKey,
+        tags: assertion.tags,
       });
       await myIdentity.controlPipeline.writer.write({ credential: { credential } });
     }
@@ -360,6 +457,8 @@ export class SpacesServiceImpl implements SpacesService {
         }),
       ),
       creator: space.inner.spaceState.creator?.key,
+      tags: space.tags,
+      membershipPolicy: space.membershipPolicy,
       cache: space.cache,
       metrics: space.metrics,
       edgeReplication: space.getEdgeReplicationSetting(),
@@ -368,9 +467,9 @@ export class SpacesServiceImpl implements SpacesService {
 
   private _requireIdentity() {
     if (!this._identityManager.identity) {
-      throw new IdentityNotInitializedError(
-        'This device has no HALO identity available. See https://docs.dxos.org/guide/platform/halo',
-      );
+      throw new IdentityNotInitializedError({
+        message: 'This device has no HALO identity available. See https://docs.dxos.org/guide/platform/halo',
+      });
     }
     return this._identityManager.identity;
   }

package/src/packlets/storage/storage.ts

@@ -14,16 +14,16 @@ import StorageDriver = Runtime.Client.Storage.StorageDriver;
 export const createStorageObjects = (config: Runtime.Client.Storage) => {
   const { persistent = false, keyStore, dataStore } = config ?? {};
   if (persistent && dataStore === StorageDriver.RAM) {
-    throw new InvalidConfigError('RAM storage cannot be used in persistent mode.');
+    throw new InvalidConfigError({ message: 'RAM storage cannot be used in persistent mode.' });
   }
   if (!persistent && dataStore !== undefined && dataStore !== StorageDriver.RAM) {
-    throw new InvalidConfigError('Cannot use a persistent storage in not persistent mode.');
+    throw new InvalidConfigError({ message: 'Cannot use a persistent storage in not persistent mode.' });
   }
   if (persistent && keyStore === StorageDriver.RAM) {
-    throw new InvalidConfigError('RAM key storage cannot be used in persistent mode.');
+    throw new InvalidConfigError({ message: 'RAM key storage cannot be used in persistent mode.' });
   }
   if (!persistent && keyStore !== StorageDriver.RAM && keyStore !== undefined) {
-    throw new InvalidConfigError('Cannot use a persistent key storage in not persistent mode.');
+    throw new InvalidConfigError({ message: 'Cannot use a persistent key storage in not persistent mode.' });
   }
 
   return {

package/src/packlets/testing/invitation-utils.ts

@@ -4,6 +4,7 @@
 
 import { Trigger } from '@dxos/async';
 import { type AuthenticatingInvitation, type CancellableInvitation, InvitationEncoder } from '@dxos/client-protocol';
+import { Context } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type DeviceProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
@@ -36,15 +37,15 @@ export type PerformInvitationCallbacks<T> = {
   onError?: (value: T) => boolean | void;
 };
 
-export type PerformInvitationParams = {
+export type PerformInvitationProps = {
   host: ServiceContext | InvitationHost;
   guest: ServiceContext | InvitationGuest;
+  guestDeviceProfile?: DeviceProfileDocument;
   options?: Partial<Invitation>;
   hooks?: {
     host?: PerformInvitationCallbacks<CancellableInvitation>;
     guest?: PerformInvitationCallbacks<AuthenticatingInvitation>;
   };
-  guestDeviceProfile?: DeviceProfileDocument;
   codeInputDelay?: number;
 };
 
@@ -52,14 +53,17 @@ export type Result = { invitation?: Invitation; error?: Error };
 
 // TODO(burdon): Make async.
 // TODO(burdon): Rename startInvitation.
+/**
+ *
+ */
 export const performInvitation = ({
   host,
   guest,
+  guestDeviceProfile,
   options,
   hooks,
-  guestDeviceProfile,
   codeInputDelay,
-}: PerformInvitationParams): [Promise<Result>, Promise<Result>] => {
+}: PerformInvitationProps): [Promise<Result>, Promise<Result>] => {
   let guestError = false;
   let guestConnected = false;
   let wereConnected = false;
@@ -226,7 +230,7 @@ export const createInvitation = async (
   };
 
   if (host instanceof ServiceContext) {
-    return host.invitationsManager.createInvitation({
+    return host.invitationsManager.createInvitation(new Context(), {
       kind: Invitation.Kind.SPACE,
       ...options,
     });
@@ -243,7 +247,7 @@ export const acceptInvitation = (
   invitation = sanitizeInvitation(invitation);
 
   if (guest instanceof ServiceContext) {
-    return guest.invitationsManager.acceptInvitation({
+    return guest.invitationsManager.acceptInvitation(new Context(), {
       invitation,
       deviceProfile: guestDeviceProfile,
     });

package/src/packlets/testing/test-builder.ts

@@ -2,6 +2,10 @@
 // Copyright 2022 DXOS.org
 //
 
+import * as Reactivity from '@effect/experimental/Reactivity';
+import * as Layer from 'effect/Layer';
+import * as ManagedRuntime from 'effect/ManagedRuntime';
+
 import { type Config } from '@dxos/config';
 import { Context } from '@dxos/context';
 import { CredentialGenerator, createCredentialSignerWithChain } from '@dxos/credentials';
@@ -15,11 +19,13 @@ import { MemorySignalManager, MemorySignalManagerContext, type SignalManager } f
 import { MemoryTransportFactory, SwarmNetworkManager } from '@dxos/network-manager';
 import { Invitation } from '@dxos/protocols/proto/dxos/client/services';
 import { type Storage, StorageType, createStorage } from '@dxos/random-access-storage';
+import { layerMemory as sqliteLayerMemory } from '@dxos/sql-sqlite/platform';
+import * as SqlTransaction from '@dxos/sql-sqlite/SqlTransaction';
 import { BlobStore } from '@dxos/teleport-extension-object-sync';
 
 import { InvitationsHandler, InvitationsManager, SpaceInvitationProtocol } from '../invitations';
-import { ClientServicesHost, ServiceContext, type ServiceContextRuntimeParams } from '../services';
-import { DataSpaceManager, type DataSpaceManagerRuntimeParams, type SigningContext } from '../spaces';
+import { ClientServicesHost, ServiceContext, type ServiceContextRuntimeProps } from '../services';
+import { DataSpaceManager, type DataSpaceManagerRuntimeProps, type SigningContext } from '../spaces';
 
 //
 // TODO(burdon): Replace with test builder.
@@ -30,6 +36,11 @@ export const createServiceHost = (config: Config, signalManagerContext: MemorySi
     config,
     signalManager: new MemorySignalManager(signalManagerContext),
     transportFactory: MemoryTransportFactory,
+    runtime: ManagedRuntime.make(
+      SqlTransaction.layer
+        .pipe(Layer.provideMerge(sqliteLayerMemory), Layer.provideMerge(Reactivity.layer))
+        .pipe(Layer.orDie),
+    ).runtimeEffect,
   });
 };
 
@@ -39,11 +50,11 @@ export const createServiceContext = async ({
     return new MemorySignalManager(signalContext);
   },
   storage = createStorage({ type: StorageType.RAM }),
-  runtimeParams,
+  runtimeProps,
 }: {
   signalManagerFactory?: () => Promise<SignalManager>;
   storage?: Storage;
-  runtimeParams?: ServiceContextRuntimeParams;
+  runtimeProps?: ServiceContextRuntimeProps;
 } = {}) => {
   const signalManager = await signalManagerFactory();
   const networkManager = new SwarmNetworkManager({
@@ -53,9 +64,15 @@ export const createServiceContext = async ({
   const level = createTestLevel();
   await level.open();
 
-  return new ServiceContext(storage, level, networkManager, signalManager, undefined, undefined, {
-    invitationConnectionDefaultParams: { teleport: { controlHeartbeatInterval: 200 } },
-    ...runtimeParams,
+  const runtime = ManagedRuntime.make(
+    SqlTransaction.layer
+      .pipe(Layer.provideMerge(sqliteLayerMemory), Layer.provideMerge(Reactivity.layer))
+      .pipe(Layer.orDie),
+  ).runtimeEffect;
+
+  return new ServiceContext(storage, level, networkManager, signalManager, undefined, undefined, runtime, {
+    invitationConnectionDefaultProps: { teleport: { controlHeartbeatInterval: 200 } },
+    ...runtimeProps,
   });
 };
 
@@ -95,7 +112,7 @@ export class TestBuilder {
 
 export type TestPeerOpts = {
   dataStore?: StorageType;
-  dataSpaceParams?: DataSpaceManagerRuntimeParams;
+  dataSpaceProps?: DataSpaceManagerRuntimeProps;
 };
 
 export type TestPeerProps = {
@@ -116,6 +133,11 @@ export type TestPeerProps = {
 
 export class TestPeer {
   private _props: TestPeerProps = {};
+  private readonly _runtime = ManagedRuntime.make(
+    SqlTransaction.layer
+      .pipe(Layer.provideMerge(sqliteLayerMemory), Layer.provideMerge(Reactivity.layer))
+      .pipe(Layer.orDie),
+  );
 
   constructor(
     private readonly _signalContext: MemorySignalManagerContext,
@@ -179,7 +201,10 @@ export class TestPeer {
   }
 
   get echoHost() {
-    return (this._props.echoHost ??= new EchoHost({ kv: this.level }));
+    return (this._props.echoHost ??= new EchoHost({
+      kv: this.level,
+      runtime: this._runtime.runtimeEffect,
+    }));
   }
 
   get meshEchoReplicator() {
@@ -198,7 +223,7 @@ export class TestPeer {
       edgeConnection: undefined,
       meshReplicator: this.meshEchoReplicator,
       echoEdgeReplicator: undefined,
-      runtimeParams: this._opts.dataSpaceParams,
+      runtimeProps: this._opts.dataSpaceProps,
     }));
   }
 
@@ -227,6 +252,7 @@ export class TestPeer {
   async destroy(): Promise<void> {
     await this.level.close();
     await this.storage.reset();
+    await this._runtime.dispose();
   }
 }