@dxos/echo-pipeline 0.1.24-next.6b1f434

package/src/space/space.test.ts

@@ -0,0 +1,144 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import expect from 'expect';
+import assert from 'node:assert';
+
+import { CredentialGenerator } from '@dxos/credentials';
+import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { describe, test, afterTest } from '@dxos/test';
+
+import { TestAgentBuilder, testLocalDatabase } from '../testing';
+
+// TODO(burdon): Factor out?
+const run = <T>(cb: () => Promise<T>): Promise<T> => cb();
+
+describe('space/space', () => {
+  test('crates a database with object model', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const agent = await builder.createPeer();
+    const [space, db] = await agent.createSpace();
+
+    await space.open();
+    expect(space.isOpen).toBeTruthy();
+    afterTest(() => space.close());
+
+    {
+      // Genesis
+      const generator = new CredentialGenerator(agent.keyring, agent.identityKey, agent.deviceKey);
+      const credentials = [
+        ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey!)),
+        await generator.createFeedAdmission(space.key, space.dataFeedKey!, AdmittedFeed.Designation.DATA)
+      ];
+
+      for (const credential of credentials) {
+        await space.controlPipeline.writer.write({
+          credential: { credential }
+        });
+      }
+
+      // TODO(burdon): Debugging only.
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+    }
+
+    assert(db.databaseBackend);
+    await testLocalDatabase(db);
+
+    await builder.close();
+    expect(space.isOpen).toBeFalsy();
+  });
+
+  test('two spaces replicating', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+
+    //
+    // Agent 1
+    //
+    const [agent1, space1, db1] = await run(async () => {
+      const agent = await builder.createPeer();
+      const [space, db] = await agent.createSpace(agent.identityKey);
+
+      await space.open();
+      expect(space.isOpen).toBeTruthy();
+      afterTest(() => space.close());
+
+      {
+        // Genesis
+        const generator = new CredentialGenerator(agent.keyring, agent.identityKey, agent.deviceKey);
+        const credentials = [
+          ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey!)),
+          await generator.createFeedAdmission(space.key, space.dataFeedKey!, AdmittedFeed.Designation.DATA)
+        ];
+
+        for (const credential of credentials) {
+          await space.controlPipeline.writer.write({
+            credential: { credential }
+          });
+        }
+
+        await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+      }
+
+      return [agent, space, db];
+    });
+
+    //
+    // Agent 2
+    //
+    const [agent2, space2, db2] = await run(async () => {
+      // NOTE: The genesisKey would be passed as part of the invitation.
+      const agent = await builder.createPeer();
+      const [space, db] = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey);
+
+      await space.open();
+      expect(space.isOpen).toBeTruthy();
+      afterTest(() => space.close());
+
+      return [agent, space, db];
+    });
+
+    expect(agent1).toBeDefined();
+    expect(agent2).toBeDefined();
+
+    {
+      // Write invitation from agent 1.
+      const generator = new CredentialGenerator(agent1.keyring, agent1.identityKey, agent1.deviceKey);
+      const credentials = await generator.createMemberInvitation(
+        space1.key,
+        agent2.identityKey,
+        agent2.deviceKey,
+        space2.controlFeedKey!,
+        space2.dataFeedKey!,
+        space1.genesisFeedKey
+      );
+
+      for (const credential of credentials) {
+        await space1.controlPipeline.writer.write({
+          credential: { credential }
+        });
+      }
+    }
+
+    {
+      // Initial data exchange.
+
+      // Agent 1 reads all feed messages.
+      await space1.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+
+      // Agent 2 reads all feed messages.
+      await space2.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+    }
+
+    // TODO(burdon): Write multiple items (extract for all tests).
+
+    await testLocalDatabase(db1, db2);
+    await testLocalDatabase(db2, db1);
+
+    await builder.close();
+    expect(space1.isOpen).toBeFalsy();
+    expect(space2.isOpen).toBeFalsy();
+  });
+});

package/src/space/space.ts

@@ -0,0 +1,199 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { Event, synchronized, trackLeaks } from '@dxos/async';
+import { FeedInfo } from '@dxos/credentials';
+import { FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log, logInfo } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { AsyncCallback, Callback } from '@dxos/util';
+
+import { Pipeline, PipelineAccessor } from '../pipeline';
+import { ControlPipeline } from './control-pipeline';
+import { DataPipelineController } from './data-pipeline-controller';
+import { SpaceProtocol } from './space-protocol';
+
+// TODO(burdon): Factor out?
+type FeedProvider = (feedKey: PublicKey) => Promise<FeedWrapper<FeedMessage>>;
+
+export type SpaceParams = {
+  spaceKey: PublicKey;
+  protocol: SpaceProtocol;
+  genesisFeed: FeedWrapper<FeedMessage>;
+  feedProvider: FeedProvider;
+};
+
+/**
+ * Spaces are globally addressable databases with access control.
+ */
+// TODO(dmaretskyi): Extract database stuff.
+@trackLeaks('open', 'close')
+export class Space {
+  public readonly onCredentialProcessed = new Callback<AsyncCallback<Credential>>();
+  public readonly stateUpdate = new Event();
+  public readonly protocol: SpaceProtocol;
+
+  private readonly _key: PublicKey;
+  private readonly _genesisFeedKey: PublicKey;
+  private readonly _feedProvider: FeedProvider;
+  private readonly _controlPipeline: ControlPipeline;
+
+  private _isOpen = false;
+  private _controlFeed?: FeedWrapper<FeedMessage>;
+  private _dataFeed?: FeedWrapper<FeedMessage>;
+  private _dataPipeline?: Pipeline;
+  private _dataPipelineController?: DataPipelineController;
+
+  constructor({ spaceKey, protocol, genesisFeed, feedProvider }: SpaceParams) {
+    assert(spaceKey && feedProvider);
+    this._key = spaceKey;
+    this._genesisFeedKey = genesisFeed.key;
+    this._feedProvider = feedProvider;
+
+    this._controlPipeline = new ControlPipeline({ spaceKey, genesisFeed, feedProvider });
+    this._controlPipeline.onFeedAdmitted.set(async (info) => {
+      if (info.assertion.designation === AdmittedFeed.Designation.DATA) {
+        // We will add all existing data feeds when the data pipeline is initialized.
+        if (this._dataPipeline) {
+          await this._dataPipeline.addFeed(await feedProvider(info.key));
+        }
+      }
+
+      if (!info.key.equals(genesisFeed.key)) {
+        this.protocol.addFeed(await feedProvider(info.key));
+      }
+    });
+
+    this._controlPipeline.onCredentialProcessed.set(async (credential) => {
+      await this.onCredentialProcessed.callIfSet(credential);
+      log('onCredentialProcessed', { credential });
+      this.stateUpdate.emit();
+    });
+
+    // Start replicating the genesis feed.
+    this.protocol = protocol;
+    this.protocol.addFeed(genesisFeed);
+  }
+
+  @logInfo
+  get key() {
+    return this._key;
+  }
+
+  get isOpen() {
+    return this._isOpen;
+  }
+
+  get genesisFeedKey(): PublicKey {
+    return this._genesisFeedKey;
+  }
+
+  get controlFeedKey() {
+    return this._controlFeed?.key;
+  }
+
+  get dataFeedKey() {
+    return this._dataFeed?.key;
+  }
+
+  get spaceState() {
+    return this._controlPipeline.spaceState;
+  }
+
+  /**
+   * @test-only
+   */
+  get controlPipeline(): PipelineAccessor {
+    return this._controlPipeline.pipeline;
+  }
+
+  setControlFeed(feed: FeedWrapper<FeedMessage>) {
+    assert(!this._controlFeed, 'Control feed already set.');
+    this._controlFeed = feed;
+    this._controlPipeline.setWriteFeed(feed);
+    return this;
+  }
+
+  setDataFeed(feed: FeedWrapper<FeedMessage>) {
+    assert(!this._dataFeed, 'Data feed already set.');
+    this._dataFeed = feed;
+    this._dataPipeline?.setWriteFeed(feed);
+    return this;
+  }
+
+  /**
+   * Use for diagnostics.
+   */
+  getControlFeeds(): FeedInfo[] {
+    return Array.from(this._controlPipeline.spaceState.feeds.values());
+  }
+
+  /**
+   * Use for diagnostics.
+   */
+  // getDataFeeds(): FeedInfo[] {
+  //   return this._dataPipeline?.getFeeds();
+  // }
+
+  @synchronized
+  async open() {
+    log('opening...');
+    if (this._isOpen) {
+      return;
+    }
+
+    // Order is important.
+    await this._controlPipeline.start();
+    await this.protocol.start();
+
+    this._isOpen = true;
+    log('opened');
+  }
+
+  @synchronized
+  async close() {
+    log('closing...', { key: this._key });
+    if (!this._isOpen) {
+      return;
+    }
+
+    // Closes in reverse order to open.
+    await this.protocol.stop();
+    await this._dataPipelineController?.close();
+    await this._controlPipeline.stop();
+
+    this._isOpen = false;
+    log('closed');
+  }
+
+  async initDataPipeline(controller: DataPipelineController) {
+    assert(this._isOpen, 'Space must be open to initialize data pipeline.');
+    assert(!this._dataPipelineController, 'Data pipeline already initialized.');
+    assert(!this._dataPipeline, 'Data pipeline already initialized.');
+
+    this._dataPipelineController = controller;
+    await this._dataPipelineController.open({
+      openPipeline: async (start) => {
+        assert(!this._dataPipeline, 'Data pipeline already initialized.'); // TODO(dmaretskyi): Allow concurrent pipelines.
+        // Create pipeline.
+        this._dataPipeline = new Pipeline(start);
+        if (this._dataFeed) {
+          this._dataPipeline.setWriteFeed(this._dataFeed);
+        }
+        for (const feed of this._controlPipeline.spaceState.feeds.values()) {
+          if (feed.assertion.designation === AdmittedFeed.Designation.DATA) {
+            await this._dataPipeline.addFeed(await this._feedProvider(feed.key));
+          }
+        }
+
+        await this._dataPipeline.start();
+        return this._dataPipeline;
+      }
+    });
+  }
+}

package/src/testing/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './test-agent-builder';
+export * from './test-feed-builder';
+export * from './util';

package/src/testing/test-agent-builder.ts

@@ -0,0 +1,193 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { DocumentModel } from '@dxos/document-model';
+import { FeedStore } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import { PublicKey } from '@dxos/keys';
+import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
+import { ModelFactory } from '@dxos/model-factory';
+import { createWebRTCTransportFactory, MemoryTransportFactory, NetworkManager } from '@dxos/network-manager';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { createStorage, Storage, StorageType } from '@dxos/random-access-storage';
+import { Presence } from '@dxos/teleport-extension-presence';
+import { ComplexMap } from '@dxos/util';
+
+import { SnapshotManager, SnapshotStore } from '../dbhost';
+import { MetadataStore } from '../metadata';
+import { MOCK_AUTH_PROVIDER, MOCK_AUTH_VERIFIER, Space, SpaceManager, SpaceProtocol } from '../space';
+import { DataPipelineControllerImpl } from '../space/data-pipeline-controller';
+import { TestFeedBuilder } from './test-feed-builder';
+
+export type NetworkManagerProvider = () => NetworkManager;
+
+export const MemoryNetworkManagerProvider =
+  (signalContext: MemorySignalManagerContext): NetworkManagerProvider =>
+  () =>
+    new NetworkManager({
+      signalManager: new MemorySignalManager(signalContext),
+      transportFactory: MemoryTransportFactory
+    });
+
+export const WebsocketNetworkManagerProvider =
+  (signalUrl: string): NetworkManagerProvider =>
+  () =>
+    new NetworkManager({
+      signalManager: new WebsocketSignalManager([signalUrl]),
+      transportFactory: createWebRTCTransportFactory()
+    });
+
+export type TestAgentBuilderOptions = {
+  storage?: Storage;
+  networkManagerProvider?: NetworkManagerProvider;
+};
+
+/**
+ * Factory for test agents.
+ */
+export class TestAgentBuilder {
+  private readonly _agents = new ComplexMap<PublicKey, TestAgent>(PublicKey.hash);
+  private readonly _storage: Storage;
+  private readonly _networkManagerProvider: NetworkManagerProvider;
+
+  constructor({ storage, networkManagerProvider }: TestAgentBuilderOptions = {}) {
+    this._storage = storage ?? createStorage({ type: StorageType.RAM });
+    this._networkManagerProvider =
+      networkManagerProvider ?? MemoryNetworkManagerProvider(new MemorySignalManagerContext());
+  }
+
+  async close() {
+    return Promise.all(this.agents.map((agent) => agent.close()));
+  }
+
+  get agents() {
+    return Array.from(this._agents.values());
+  }
+
+  getAgent(deviceKey: PublicKey) {
+    return this._agents.get(deviceKey);
+  }
+
+  async createPeer(): Promise<TestAgent> {
+    // prettier-ignore
+    const feedBuilder = new TestFeedBuilder()
+      .setStorage(this._storage, `agent-${this._agents.size}`);
+
+    const identityKey = await feedBuilder.keyring.createKey();
+    const deviceKey = await feedBuilder.keyring.createKey();
+
+    const agent = new TestAgent(this._networkManagerProvider, feedBuilder, identityKey, deviceKey);
+    this._agents.set(deviceKey, agent);
+    return agent;
+  }
+}
+
+/**
+ * Test agent that enables the creation and replication of multiple spaces.
+ */
+export class TestAgent {
+  private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
+
+  public readonly keyring: Keyring;
+  public readonly feedStore: FeedStore<FeedMessage>;
+
+  constructor(
+    private readonly _networkManagerProvider: NetworkManagerProvider,
+    private readonly _feedBuilder: TestFeedBuilder,
+    public readonly identityKey: PublicKey,
+    public readonly deviceKey: PublicKey
+  ) {
+    this.keyring = this._feedBuilder.keyring;
+    this.feedStore = this._feedBuilder.createFeedStore();
+  }
+
+  async close() {
+    return Promise.all([...this.spaces.map((space) => space.close())]);
+  }
+
+  get spaces() {
+    return Array.from(this._spaces.values());
+  }
+
+  getSpace(spaceKey: PublicKey) {
+    return this._spaces.get(spaceKey);
+  }
+
+  createSpaceManager() {
+    return new SpaceManager({
+      feedStore: this._feedBuilder.createFeedStore(),
+      networkManager: this._networkManagerProvider()
+    });
+  }
+
+  async createSpace(
+    identityKey: PublicKey = this.identityKey,
+    spaceKey?: PublicKey,
+    genesisKey?: PublicKey
+  ): Promise<[Space, DataPipelineControllerImpl]> {
+    if (!spaceKey) {
+      spaceKey = await this.keyring.createKey();
+    }
+
+    const dataFeedKey = await this.keyring.createKey();
+    const dataFeed = await this.feedStore.openFeed(dataFeedKey, { writable: true });
+
+    const controlFeedKey = await this.keyring.createKey();
+    const controlFeed = await this.feedStore.openFeed(controlFeedKey, { writable: true });
+    const genesisFeed = genesisKey ? await this.feedStore.openFeed(genesisKey) : controlFeed;
+    const snapshotManager = new SnapshotManager(new SnapshotStore(createStorage().createDirectory('snapshots')));
+
+    const metadataStore = new MetadataStore(createStorage().createDirectory('metadata'));
+    await metadataStore.addSpace({ key: spaceKey });
+    const dataPipelineController: DataPipelineControllerImpl = new DataPipelineControllerImpl({
+      modelFactory: new ModelFactory().registerModel(DocumentModel),
+      metadataStore,
+      snapshotManager,
+      memberKey: identityKey,
+      spaceKey,
+      feedInfoProvider: (feedKey) => space.spaceState.feeds.get(feedKey),
+      snapshotId: undefined
+    });
+    const space = new Space({
+      spaceKey,
+      protocol: this.createSpaceProtocol(spaceKey),
+      genesisFeed,
+      feedProvider: (feedKey) => this.feedStore.openFeed(feedKey)
+    })
+      .setControlFeed(controlFeed)
+      .setDataFeed(dataFeed);
+    await space.open();
+    await space.initDataPipeline(dataPipelineController);
+
+    this._spaces.set(spaceKey, space);
+    return [space, dataPipelineController];
+  }
+
+  createSpaceProtocol(topic: PublicKey, presence?: Presence) {
+    return new SpaceProtocol({
+      topic,
+      swarmIdentity: {
+        peerKey: this.deviceKey,
+        credentialProvider: MOCK_AUTH_PROVIDER,
+        credentialAuthenticator: MOCK_AUTH_VERIFIER
+      },
+      networkManager: this._networkManagerProvider(),
+      onSessionAuth: (session) => {
+        session.addExtension(
+          'dxos.mesh.teleport.presence',
+          (presence ?? this.createPresence()).createExtension({ remotePeerId: session.remotePeerId })
+        );
+      }
+    });
+  }
+
+  createPresence() {
+    return new Presence({
+      localPeerId: this.deviceKey,
+      announceInterval: 30,
+      offlineTimeout: 200,
+      identityKey: this.identityKey
+    });
+  }
+}

package/src/testing/test-feed-builder.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { TestBuilder } from '@dxos/feed-store/testing';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+
+import { valueEncoding } from '../common';
+
+/**
+ * Builder with default encoder and generator.
+ */
+export class TestFeedBuilder extends TestBuilder<FeedMessage> {
+  constructor() {
+    super({
+      valueEncoding
+    });
+  }
+}

package/src/testing/util.ts

@@ -0,0 +1,77 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { asyncTimeout } from '@dxos/async';
+import { DocumentModel } from '@dxos/document-model';
+import { DatabaseBackendProxy, ItemManager } from '@dxos/echo-db';
+import { MockFeedWriter } from '@dxos/feed-store/testing';
+import { PublicKey } from '@dxos/keys';
+import { ModelFactory } from '@dxos/model-factory';
+import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { Timeframe } from '@dxos/timeframe';
+
+import { DatabaseBackendHost, DataServiceHost, DataServiceImpl, DataServiceSubscriptions } from '../dbhost';
+import { DataPipelineControllerImpl } from '../space';
+
+export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
+  const feed = new MockFeedWriter<DataMessage>();
+  const backend = new DatabaseBackendHost(feed, undefined, { snapshots: true });
+
+  feed.written.on(([data, meta]) =>
+    backend.echoProcessor({
+      data: data.object,
+      meta: {
+        ...meta,
+        memberKey: PublicKey.random(),
+        timeframe: new Timeframe([[meta.feedKey, meta.seq]])
+      }
+    })
+  );
+
+  const itemManager = new ItemManager(modelFactory);
+  await backend.open(itemManager, new ModelFactory().registerModel(DocumentModel));
+  return {
+    backend,
+    itemManager
+  };
+};
+
+export const createRemoteDatabaseFromDataServiceHost = async (
+  modelFactory: ModelFactory,
+  dataServiceHost: DataServiceHost
+) => {
+  const dataServiceSubscriptions = new DataServiceSubscriptions();
+  const dataService = new DataServiceImpl(dataServiceSubscriptions);
+
+  const spaceKey = PublicKey.random();
+  dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
+
+  const backend = new DatabaseBackendProxy(dataService, spaceKey);
+  const itemManager = new ItemManager(modelFactory);
+  await backend.open(itemManager, new ModelFactory().registerModel(DocumentModel));
+  return {
+    itemManager,
+    backend
+  };
+};
+
+export const testLocalDatabase = async (
+  create: DataPipelineControllerImpl,
+  check: DataPipelineControllerImpl = create
+) => {
+  const objectId = PublicKey.random().toHex();
+  await create.databaseBackend!.getWriteStream()?.write({
+    object: {
+      objectId,
+      genesis: {
+        modelType: DocumentModel.meta.type
+      }
+    }
+  });
+
+  await asyncTimeout(
+    check._itemManager.update.waitForCondition(() => check._itemManager.entities.has(objectId)),
+    500
+  );
+};

package/src/tests/database.test.ts

@@ -0,0 +1,53 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import expect from 'expect';
+
+import { DocumentModel, MutationBuilder } from '@dxos/document-model';
+import { createModelMutation, encodeModelMutation, genesisMutation } from '@dxos/echo-db';
+import { PublicKey } from '@dxos/keys';
+import { ModelFactory } from '@dxos/model-factory';
+import { test } from '@dxos/test';
+
+import { createMemoryDatabase, createRemoteDatabaseFromDataServiceHost } from '../testing';
+
+const createDatabase = async () => {
+  // prettier-ignore
+  const modelFactory = new ModelFactory()
+    .registerModel(DocumentModel);
+
+  // TODO(dmaretskyi): Fix.
+  const host = await createMemoryDatabase(modelFactory);
+  const proxy = await createRemoteDatabaseFromDataServiceHost(modelFactory, host.backend.createDataServiceHost());
+  return proxy;
+};
+
+describe('database', () => {
+  describe('proxy-service mode', () => {
+    test('create object', async () => {
+      const database = await createDatabase();
+
+      const result = database.backend.mutate(genesisMutation(PublicKey.random().toHex(), DocumentModel.meta.type));
+      expect(result.objectsCreated.length).toEqual(1);
+      expect(database.itemManager.entities.has(result.objectsCreated[0].id));
+
+      await (await result.getReceipt()).waitToBeProcessed();
+      expect(database.itemManager.entities.has(result.objectsCreated[0].id));
+    });
+
+    test('mutate document', async () => {
+      const database = await createDatabase();
+
+      const id = PublicKey.random().toHex();
+      database.backend.mutate(genesisMutation(id, DocumentModel.meta.type));
+      const result = database.backend.mutate(
+        createModelMutation(id, encodeModelMutation(DocumentModel.meta, new MutationBuilder().set('test', 42).build()))
+      );
+      expect(database.itemManager.getItem(id)!.state.data.test).toEqual(42);
+
+      await (await result.getReceipt()).waitToBeProcessed();
+      expect(database.itemManager.getItem(id)!.state.data.test).toEqual(42);
+    });
+  });
+});