@dxos/client-services 0.10.0 → 0.11.0

package/src/packlets/space/testing/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from '../../pipeline/testing';
+export * from './change-metadata';
+export * from './test-agent-builder';

package/src/packlets/space/testing/test-agent-builder.ts

@@ -0,0 +1,252 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { Context } from '@dxos/context';
+import { CredentialGenerator } from '@dxos/credentials';
+import { type FeedStore } from '@dxos/feed-store';
+import { type Keyring } from '@dxos/keyring';
+import { PublicKey } from '@dxos/keys';
+import { MemorySignalManager, MemorySignalManagerContext } from '@dxos/messaging';
+import { MemoryTransportFactory, SwarmNetworkManager } from '@dxos/network-manager';
+import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { type SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { type Storage, StorageType, createStorage } from '@dxos/random-access-storage';
+import { Gossip, Presence } from '@dxos/teleport-extension-gossip';
+import { BlobStore } from '@dxos/teleport-extension-object-sync';
+import { ComplexMap } from '@dxos/util';
+
+import { MetadataStore } from '../../metadata';
+import { TestFeedBuilder } from '../../pipeline/testing';
+import { MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, type Space, SpaceManager, SpaceProtocol } from '../index';
+
+export type NetworkManagerProvider = () => SwarmNetworkManager;
+
+export const MemoryNetworkManagerProvider =
+  (signalContext: MemorySignalManagerContext): NetworkManagerProvider =>
+  () =>
+    new SwarmNetworkManager({
+      signalManager: new MemorySignalManager(signalContext),
+      transportFactory: MemoryTransportFactory,
+    });
+
+export type TestAgentBuilderOptions = {
+  storage?: Storage;
+  networkManagerProvider?: NetworkManagerProvider;
+};
+
+/**
+ * Factory for test agents.
+ */
+export class TestAgentBuilder {
+  private readonly _agents = new ComplexMap<PublicKey, TestAgent>(PublicKey.hash);
+  private readonly _storage: Storage;
+  private readonly _networkManagerProvider: NetworkManagerProvider;
+
+  constructor({ storage, networkManagerProvider }: TestAgentBuilderOptions = {}) {
+    this._storage = storage ?? createStorage({ type: StorageType.RAM });
+    this._networkManagerProvider =
+      networkManagerProvider ?? MemoryNetworkManagerProvider(new MemorySignalManagerContext());
+  }
+
+  async close() {
+    return Promise.all(this.agents.map((agent) => agent.close()));
+  }
+
+  get agents() {
+    return Array.from(this._agents.values());
+  }
+
+  getAgent(deviceKey: PublicKey): TestAgent | undefined {
+    return this._agents.get(deviceKey);
+  }
+
+  async createPeer(): Promise<TestAgent> {
+    // prettier-ignore
+    const feedBuilder = new TestFeedBuilder()
+      .setStorage(this._storage, `agent-${this._agents.size}`);
+
+    const identityKey = await feedBuilder.keyring.createKey();
+    const deviceKey = await feedBuilder.keyring.createKey();
+
+    const agent = new TestAgent(this._networkManagerProvider, feedBuilder, identityKey, deviceKey);
+    this._agents.set(deviceKey, agent);
+    return agent;
+  }
+}
+
+/**
+ * Test agent that enables the creation and replication of multiple spaces.
+ */
+export class TestAgent {
+  private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
+
+  public readonly storage: Storage;
+  public readonly keyring: Keyring;
+  public readonly feedStore: FeedStore<FeedMessage>;
+
+  private _metadataStore?: MetadataStore;
+  get metadataStore() {
+    return (this._metadataStore ??= new MetadataStore(this.storage.createDirectory('metadata')));
+  }
+
+  private _blobStore?: BlobStore;
+  get blobStore() {
+    return (this._blobStore ??= new BlobStore(this.storage.createDirectory('blobs')));
+  }
+
+  constructor(
+    private readonly _networkManagerProvider: NetworkManagerProvider,
+    private readonly _feedBuilder: TestFeedBuilder,
+    public readonly identityKey: PublicKey,
+    public readonly deviceKey: PublicKey,
+  ) {
+    this.storage = this._feedBuilder.storage;
+    this.keyring = this._feedBuilder.keyring;
+    this.feedStore = this._feedBuilder.createFeedStore();
+  }
+
+  async close() {
+    return Promise.all([...this.spaces.map((space) => space.close())]);
+  }
+
+  get spaces() {
+    return Array.from(this._spaces.values());
+  }
+
+  getSpace(spaceKey: PublicKey): Space | undefined {
+    return this._spaces.get(spaceKey);
+  }
+
+  private _networkManager?: SwarmNetworkManager;
+  get networkManager() {
+    if (this._networkManager) {
+      return this._networkManager;
+    }
+
+    this._networkManager = this._networkManagerProvider();
+    this._networkManager.setPeerInfo({ peerKey: this.deviceKey.toHex(), identityKey: this.identityKey.toHex() });
+
+    return this._networkManager;
+  }
+
+  private _spaceManager?: SpaceManager;
+  get spaceManager() {
+    return (this._spaceManager ??= new SpaceManager({
+      feedStore: this.feedStore,
+      networkManager: this.networkManager,
+      metadataStore: this.metadataStore,
+      blobStore: this.blobStore,
+    }));
+  }
+
+  async createSpace(
+    identityKey: PublicKey = this.identityKey,
+    spaceKey?: PublicKey,
+    genesisKey?: PublicKey,
+    dataKey?: PublicKey,
+    saveMetadata = false,
+  ): Promise<Space> {
+    if (!spaceKey) {
+      saveMetadata = true;
+      spaceKey = await this.keyring.createKey();
+    }
+    if (!genesisKey) {
+      genesisKey = await this.keyring.createKey();
+    }
+
+    const controlFeed = await this.feedStore.openFeed(genesisKey, { writable: true });
+    const dataFeed = await this.feedStore.openFeed(dataKey ?? (await this.keyring.createKey()), {
+      writable: true,
+      sparse: true,
+    });
+
+    const metadata: SpaceMetadata = {
+      key: spaceKey,
+      genesisFeedKey: genesisKey,
+      controlFeedKey: controlFeed.key,
+      dataFeedKey: dataFeed.key,
+    };
+    if (saveMetadata) {
+      await this.metadataStore.addSpace(metadata);
+    }
+
+    await this.spaceManager.open();
+    const space = await this.spaceManager.constructSpace({
+      metadata,
+      swarmIdentity: {
+        identityKey: this.identityKey,
+        peerKey: this.deviceKey,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER,
+      },
+      memberKey: identityKey,
+      onAuthorizedConnection: (session) => {
+        session.addExtension(
+          'dxos.mesh.teleport.gossip',
+          this.createGossip().createExtension({ remotePeerId: session.remotePeerId }),
+        );
+      },
+      onDelegatedInvitationStatusChange: async () => {},
+      onMemberRolesChanged: async () => {},
+    });
+    await space.setControlFeed(controlFeed);
+    await space.setDataFeed(dataFeed);
+
+    await space.open(new Context());
+
+    this._spaces.set(spaceKey, space);
+    return space;
+  }
+
+  createSpaceProtocol(topic: PublicKey, gossip?: Gossip): SpaceProtocol {
+    return new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        identityKey: this.identityKey,
+        peerKey: this.deviceKey,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER,
+      },
+      networkManager: this.networkManager,
+      blobStore: this.blobStore,
+      onSessionAuth: (session) => {
+        session.addExtension(
+          'dxos.mesh.teleport.gossip',
+          (gossip ?? this.createGossip()).createExtension({ remotePeerId: session.remotePeerId }),
+        );
+      },
+    });
+  }
+
+  createGossip(): Gossip {
+    return new Gossip({
+      localPeerId: this.deviceKey,
+    });
+  }
+
+  createPresence(gossip?: Gossip): Presence {
+    return new Presence({
+      announceInterval: 30,
+      offlineTimeout: 200,
+      identityKey: this.identityKey,
+      gossip: gossip ?? this.createGossip(),
+    });
+  }
+
+  async spaceGenesis(space: Space): Promise<void> {
+    const generator = new CredentialGenerator(this.keyring, this.identityKey, this.deviceKey);
+    const credentials = [
+      ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey!)),
+      await generator.createFeedAdmission(space.key, space.dataFeedKey!, AdmittedFeed.Designation.DATA),
+      await generator.createEpochCredential(space.key),
+    ];
+
+    for (const credential of credentials) {
+      await space.controlPipeline.writer.write({
+        credential: { credential },
+      });
+    }
+  }
+}

package/src/packlets/space-export/serialized-space-reader.ts

@@ -107,6 +107,8 @@ export const objJsonToObjectStructure = (obj: Obj.JSON): EntityStructure => {
       // `objectStructureToObjJson`).
       ...(meta?.key !== undefined ? { key: meta.key } : {}),
       ...(meta?.version !== undefined ? { version: meta.version } : {}),
+      // Preserve annotations (see the symmetric write in `objectStructureToObjJson`).
+      ...(meta?.annotations ? { annotations: meta.annotations } : {}),
     },
     data,
   };
@@ -114,7 +116,7 @@ export const objJsonToObjectStructure = (obj: Obj.JSON): EntityStructure => {
 
 /**
  * Build a new {@link DatabaseDirectory} containing every object from the archive,
- * keyed by object id. The caller is responsible for stamping the `access.spaceKey`
+ * keyed by object id. The caller is responsible for stamping the `access` (space id/key)
  * and version fields after the document is created.
  */
 export const buildDatabaseDirectoryFromObjects = (objects: readonly Obj.JSON[]): DatabaseDirectory => {

package/src/packlets/space-export/serialized-space-writer.ts

@@ -3,17 +3,22 @@
 //
 
 import { type AutomergeUrl } from '@automerge/automerge-repo';
+import * as Effect from 'effect/Effect';
+import * as Exit from 'effect/Exit';
+import * as Scope from 'effect/Scope';
 
 import { Context } from '@dxos/context';
 import { type Obj } from '@dxos/echo';
 import { type SerializedFeed, type SerializedSpace } from '@dxos/echo-client';
 import { type EchoHost } from '@dxos/echo-host';
 import { type DatabaseDirectory, type EntityStructure } from '@dxos/echo-protocol';
+import { EffectEx } from '@dxos/effect';
 import { assertState, invariant } from '@dxos/invariant';
 import { DXN, type EntityId, type IdentityDid, type SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { FeedProtocol } from '@dxos/protocols';
+import { FeedProtocol, makeInProcessClient } from '@dxos/protocols';
 import { SpaceArchive } from '@dxos/protocols/proto/dxos/client/services';
+import { FeedService } from '@dxos/protocols/rpc';
 import { createFilename } from '@dxos/util';
 
 import { type DataSpace } from '../spaces/data-space';
@@ -164,6 +169,11 @@ export const objectStructureToObjJson = (objectId: string, structure: EntityStru
       // a type loaded from an archive would fall back to its object id.
       ...(structure.meta.key !== undefined ? { key: structure.meta.key } : {}),
       ...(structure.meta.version !== undefined ? { version: structure.meta.version } : {}),
+      // Preserve annotations (e.g. `AppAnnotation.RootCollectionAnnotation` on a space's
+      // properties object) so archives round-trip app-level metadata, not just schema data.
+      ...(structure.meta.annotations && Object.keys(structure.meta.annotations).length > 0
+        ? { annotations: structure.meta.annotations }
+        : {}),
     };
   }
   if (structure.system?.deleted) {
@@ -201,7 +211,7 @@ const exportFeedData = async (space: DataSpace, echoHost: EchoHost, objects: Obj
     const namespace = (obj as any).namespace === 'trace' ? 'trace' : 'data';
 
     try {
-      const messages = await collectQueueMessages(echoHost, spaceId, obj.id, namespace);
+      const messages = await collectFeedMessages(echoHost, spaceId, obj.id, namespace);
       if (messages.length > 0) {
         feeds.push({
           feedObjectId: obj.id,
@@ -217,44 +227,57 @@ const exportFeedData = async (space: DataSpace, echoHost: EchoHost, objects: Obj
   return feeds;
 };
 
-const collectQueueMessages = async (
+const collectFeedMessages = async (
   echoHost: EchoHost,
   spaceId: SpaceId,
-  queueId: EntityId,
+  feedId: EntityId,
   namespace: string,
 ): Promise<Obj.JSON[]> => {
-  const queuesNamespace =
+  const feedNamespace =
     namespace === 'trace' ? FeedProtocol.WellKnownNamespaces.trace : FeedProtocol.WellKnownNamespaces.data;
 
   const messages: Obj.JSON[] = [];
   let cursor: string | undefined;
-  while (true) {
-    const result = await echoHost.queuesService.queryQueue({
-      query: {
-        spaceId,
-        queueIds: [queueId],
-        queuesNamespace,
-        after: cursor,
-      },
-    });
-    const batch = (result.objects ?? []).flatMap((encoded): Obj.JSON[] => {
-      try {
-        return [JSON.parse(encoded) as Obj.JSON];
-      } catch (err) {
-        log.verbose('queue object JSON parse failed; object ignored', { encoded, error: err });
-        return [];
+
+  // Bridge the host feed Handlers to a client to call it in-process (Handlers are served, not
+  // called directly). The scope owns the client for the duration of the read.
+  const scope = Effect.runSync(Scope.make());
+  const feedClient = await EffectEx.runPromise(
+    makeInProcessClient(FeedService.Rpcs, echoHost.feedService).pipe(Effect.provideService(Scope.Scope, scope)),
+  );
+  try {
+    while (true) {
+      const result = await EffectEx.runPromise(
+        feedClient.FeedService.queryFeed({
+          query: {
+            spaceId,
+            feedIds: [feedId],
+            feedNamespace,
+            after: cursor,
+          },
+        }),
+      );
+      const batch = (result.objects ?? []).flatMap((encoded): Obj.JSON[] => {
+        try {
+          return [JSON.parse(encoded) as Obj.JSON];
+        } catch (err) {
+          log.verbose('feed object JSON parse failed; object ignored', { encoded, error: err });
+          return [];
+        }
+      });
+      if (batch.length === 0) {
+        break;
       }
-    });
-    if (batch.length === 0) {
-      break;
-    }
-    for (const message of batch) {
-      messages.push(orderObjJsonFields(message));
-    }
-    if (!result.nextCursor || result.nextCursor === cursor) {
-      break;
+      for (const message of batch) {
+        messages.push(orderObjJsonFields(message));
+      }
+      if (!result.nextCursor || result.nextCursor === cursor) {
+        break;
+      }
+      cursor = result.nextCursor;
     }
-    cursor = result.nextCursor;
+  } finally {
+    await EffectEx.runPromise(Scope.close(scope, Exit.void));
   }
   return messages;
 };

package/src/packlets/space-export/space-archive.test.ts

@@ -438,6 +438,36 @@ describe('SpaceArchive', () => {
       expect(Object.keys(ordered)).toEqual(['id', '@type', '@custom', 'data']);
     });
 
+    test('objectStructureToObjJson preserves meta.annotations', () => {
+      const id = EntityId.random();
+      const obj = objectStructureToObjJson(id, {
+        data: { name: 'Bramble Coffee Roasters' },
+        meta: { keys: [], annotations: { 'org.dxos.space.rootCollection': { '/': 'dxn:echo:@:example' } } },
+        system: { type: { '/': URI.make('dxn:org.dxos.type.spaceProperties') }, kind: 'object' },
+      });
+
+      expect((obj['@meta'] as any).annotations).toEqual({
+        'org.dxos.space.rootCollection': { '/': 'dxn:echo:@:example' },
+      });
+    });
+
+    test('buildDatabaseDirectoryFromObjects round-trips meta.annotations', () => {
+      const id = EntityId.random();
+      const objects = [
+        {
+          id,
+          '@type': 'dxn:org.dxos.type.spaceProperties',
+          '@meta': { keys: [], annotations: { 'org.dxos.space.rootCollection': { '/': 'dxn:echo:@:example' } } },
+          'name': 'Bramble Coffee Roasters',
+        },
+      ];
+      const directory = buildDatabaseDirectoryFromObjects(objects as any);
+      const structure = directory.objects![id];
+      expect(structure.meta?.annotations).toEqual({
+        'org.dxos.space.rootCollection': { '/': 'dxn:echo:@:example' },
+      });
+    });
+
     test('buildDatabaseDirectoryFromObjects flags relations', () => {
       const id = EntityId.random();
       const sourceId = EntityId.random();

package/src/packlets/spaces/data-space-manager.test.ts

@@ -7,13 +7,13 @@ import { describe, expect, test } from 'vitest';
 import { asyncTimeout, latch } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { createAdmissionCredentials } from '@dxos/credentials';
-import { AuthStatus } from '@dxos/echo-host';
 import { writeMessages } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import { openAndClose } from '@dxos/test-utils';
 
+import { AuthStatus } from '../space';
 import { TestBuilder, type TestPeer } from '../testing';
 
 describe('DataSpaceManager', () => {