@dxos/echo-pipeline 0.1.24-next.6b1f434

package/src/space/control-pipeline.test.ts

@@ -0,0 +1,162 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import expect from 'expect';
+
+import { CredentialGenerator, createCredential } from '@dxos/credentials';
+import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { createStorage, StorageType } from '@dxos/random-access-storage';
+import { describe, test, afterTest } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+
+import { valueEncoding } from '../common';
+import { ControlPipeline } from './control-pipeline';
+
+describe('space/control-pipeline', () => {
+  test('admits feeds', async () => {
+    const keyring = new Keyring();
+    const spaceKey = await keyring.createKey();
+    const identityKey = await keyring.createKey();
+    const deviceKey = await keyring.createKey();
+
+    const feedStore = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: createStorage({ type: StorageType.RAM }).createDirectory(),
+        signer: keyring,
+        hypercore: {
+          valueEncoding
+        }
+      })
+    });
+
+    const createFeed = async () => {
+      const feedKey = await keyring.createKey();
+      return feedStore.openFeed(feedKey, { writable: true });
+    };
+
+    // TODO(dmaretskyi): Separate test for cold start after genesis.
+    const genesisFeed = await createFeed();
+    const controlPipeline = new ControlPipeline({
+      spaceKey,
+      genesisFeed,
+      initialTimeframe: new Timeframe(),
+      feedProvider: (key) => feedStore.openFeed(key)
+    });
+
+    const admittedFeeds: PublicKey[] = [];
+    controlPipeline.onFeedAdmitted.set(async (info) => {
+      log.debug('feed admitted');
+      admittedFeeds.push(info.key);
+    });
+    expect(admittedFeeds).toEqual([]);
+
+    controlPipeline.setWriteFeed(genesisFeed);
+    await controlPipeline.start();
+
+    afterTest(() => controlPipeline.stop());
+
+    //
+    // Genesis
+    //
+    {
+      const generator = new CredentialGenerator(keyring, identityKey, deviceKey);
+      const credentials = await generator.createSpaceGenesis(spaceKey, genesisFeed.key);
+      expect(credentials).toHaveLength(3);
+
+      for (const credential of credentials) {
+        await controlPipeline.pipeline.writer?.write({
+          credential: {
+            credential
+          }
+        });
+      }
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key]);
+    }
+
+    // New control feed.
+    const controlFeed2 = await createFeed();
+    {
+      await controlPipeline.pipeline.writer!.write({
+        credential: {
+          credential: await createCredential({
+            signer: keyring,
+            issuer: identityKey,
+            subject: controlFeed2.key,
+            assertion: {
+              '@type': 'dxos.halo.credentials.AdmittedFeed',
+              spaceKey,
+              identityKey,
+              deviceKey,
+              designation: AdmittedFeed.Designation.CONTROL
+            }
+          })
+        }
+      });
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key]);
+    }
+
+    // New data feed.
+    const dataFeed1 = await createFeed();
+    {
+      await controlPipeline.pipeline.writer!.write({
+        credential: {
+          credential: await createCredential({
+            signer: keyring,
+            issuer: identityKey,
+            subject: dataFeed1.key,
+            assertion: {
+              '@type': 'dxos.halo.credentials.AdmittedFeed',
+              spaceKey,
+              identityKey,
+              deviceKey,
+              designation: AdmittedFeed.Designation.DATA
+            }
+          })
+        }
+      });
+
+      const end = controlPipeline.pipeline.state.endTimeframe;
+      await controlPipeline.pipeline.state.waitUntilTimeframe(end);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key, dataFeed1.key]);
+    }
+
+    // TODO(dmaretskyi): Move to other test (data feed cannot admit feeds).
+    const dataFeed2 = await createFeed();
+    {
+      await dataFeed1.append({
+        payload: {
+          '@type': 'dxos.echo.feed.FeedMessage',
+          timeframe: controlPipeline.pipeline.state.timeframe,
+          credential: {
+            credential: await createCredential({
+              signer: keyring,
+              issuer: identityKey,
+              subject: dataFeed2.key,
+              assertion: {
+                '@type': 'dxos.halo.credentials.AdmittedFeed',
+                spaceKey,
+                identityKey,
+                deviceKey,
+                designation: AdmittedFeed.Designation.DATA
+              }
+            })
+          }
+        },
+        timeframe: new Timeframe()
+      });
+
+      await controlPipeline.pipeline.state.waitUntilTimeframe(controlPipeline.pipeline.state.endTimeframe);
+      expect(admittedFeeds).toEqual([genesisFeed.key, controlFeed2.key, dataFeed1.key]);
+    }
+  });
+});

package/src/space/control-pipeline.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { SpaceStateMachine, SpaceState, MemberInfo, FeedInfo } from '@dxos/credentials';
+import { FeedWrapper } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { Timeframe } from '@dxos/timeframe';
+import { AsyncCallback, Callback } from '@dxos/util';
+
+import { Pipeline, PipelineAccessor } from '../pipeline';
+
+export type ControlPipelineParams = {
+  spaceKey: PublicKey;
+  genesisFeed: FeedWrapper<FeedMessage>;
+  feedProvider: (feedKey: PublicKey) => Promise<FeedWrapper<FeedMessage>>;
+  initialTimeframe?: Timeframe;
+};
+
+/**
+ * Processes HALO credentials, which include genesis and invitations.
+ */
+export class ControlPipeline {
+  private readonly _pipeline: Pipeline;
+  private readonly _spaceStateMachine: SpaceStateMachine;
+
+  public readonly onFeedAdmitted = new Callback<AsyncCallback<FeedInfo>>();
+  public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
+  public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
+
+  constructor({ spaceKey, genesisFeed, feedProvider, initialTimeframe }: ControlPipelineParams) {
+    this._pipeline = new Pipeline(initialTimeframe);
+    void this._pipeline.addFeed(genesisFeed); // TODO(burdon): Require async open/close?
+
+    this._spaceStateMachine = new SpaceStateMachine(spaceKey);
+    this._spaceStateMachine.onFeedAdmitted.set(async (info) => {
+      // log('feed admitted', { info });
+      log('feed admitted', { key: info.key });
+
+      // TODO(burdon): Check not stopping.
+      if (info.assertion.designation === AdmittedFeed.Designation.CONTROL && !info.key.equals(genesisFeed.key)) {
+        try {
+          const feed = await feedProvider(info.key);
+          await this._pipeline.addFeed(feed);
+        } catch (err: any) {
+          log.catch(err);
+        }
+      }
+
+      await this.onFeedAdmitted.callIfSet(info);
+    });
+
+    this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
+    this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
+  }
+
+  get spaceState(): SpaceState {
+    return this._spaceStateMachine;
+  }
+
+  get pipeline(): PipelineAccessor {
+    return this._pipeline;
+  }
+
+  setWriteFeed(feed: FeedWrapper<FeedMessage>) {
+    this._pipeline.setWriteFeed(feed);
+  }
+
+  async start() {
+    log('starting...');
+    setTimeout(async () => {
+      for await (const msg of this._pipeline.consume()) {
+        try {
+          // log('processing', { msg });
+          log('processing', { key: msg.feedKey, seq: msg.seq });
+          if (msg.data.payload.credential) {
+            const result = await this._spaceStateMachine.process(
+              msg.data.payload.credential.credential,
+              PublicKey.from(msg.feedKey)
+            );
+            if (!result) {
+              log.warn('processing failed', { msg });
+            }
+          }
+        } catch (err: any) {
+          log.catch(err);
+        }
+      }
+    });
+
+    await this._pipeline.start();
+    log('started');
+  }
+
+  async stop() {
+    log('stopping...');
+    await this._pipeline.stop();
+    log('stopped');
+  }
+}

package/src/space/data-pipeline-controller.ts

@@ -0,0 +1,234 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { Event, scheduleTask, trackLeaks } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { FeedInfo } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
+import { ItemManager } from '@dxos/echo-db';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { ModelFactory } from '@dxos/model-factory';
+import { DataMessage, FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { Timeframe } from '@dxos/timeframe';
+
+import { createMappedFeedWriter } from '../common';
+import { DatabaseBackendHost, SnapshotManager } from '../dbhost';
+import { MetadataStore } from '../metadata';
+import { Pipeline } from '../pipeline';
+
+export type DataPipelineControllerContext = {
+  openPipeline: (start: Timeframe) => Promise<Pipeline>;
+};
+
+/**
+ * Controls data pipeline in the space.
+ * Consumes mutations from the feed and applies them to the database.
+ */
+export interface DataPipelineController {
+  open(context: DataPipelineControllerContext): Promise<void>;
+  close(): Promise<void>;
+}
+
+export class NoopDataPipelineController implements DataPipelineController {
+  async open(context: DataPipelineControllerContext): Promise<void> {}
+
+  async close(): Promise<void> {}
+}
+
+export type DataPipelineControllerImplParams = {
+  modelFactory: ModelFactory;
+  snapshotManager: SnapshotManager;
+  metadataStore: MetadataStore;
+  memberKey: PublicKey;
+  spaceKey: PublicKey;
+  feedInfoProvider: (feedKey: PublicKey) => FeedInfo | undefined;
+  snapshotId: string | undefined;
+};
+
+/**
+ * Number of mutations since the last snapshot before we automatically create another snapshot.
+ */
+const MESSAGES_PER_SNAPSHOT = 10;
+
+/**
+ * Minimum time between automatic snapshots.
+ */
+const AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5000;
+
+/**
+ * Minimum time between recording latest timeframe in metadata.
+ */
+const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
+
+/**
+ * Controls data pipeline in the space.
+ * Consumes the pipeline and updates the database.
+ * Reacts to new epochs to restart the pipeline.
+ */
+@trackLeaks('open', 'close')
+export class DataPipelineControllerImpl implements DataPipelineController {
+  private _ctx = new Context();
+  private _spaceContext!: DataPipelineControllerContext;
+  private _pipeline?: Pipeline;
+  private _snapshot?: SpaceSnapshot;
+
+  private _lastAutomaticSnapshotTimeframe = new Timeframe();
+
+  public readonly onTimeframeReached = new Event<Timeframe>();
+
+  constructor(private readonly _params: DataPipelineControllerImplParams) {}
+
+  public _itemManager!: ItemManager;
+  public databaseBackend?: DatabaseBackendHost;
+
+  get pipelineState() {
+    return this._pipeline?.state;
+  }
+
+  get snapshotTimeframe() {
+    return this._snapshot?.timeframe;
+  }
+
+  getStartTimeframe(): Timeframe {
+    return snapshotTimeframeToStartingTimeframe(this.snapshotTimeframe ?? new Timeframe());
+  }
+
+  async open(spaceContext: DataPipelineControllerContext) {
+    this._spaceContext = spaceContext;
+    if (this._params.snapshotId) {
+      this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
+      this._lastAutomaticSnapshotTimeframe = this._snapshot?.timeframe ?? new Timeframe();
+    }
+
+    this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
+
+    // Create database backend.
+    const feedWriter = createMappedFeedWriter<DataMessage, FeedMessage.Payload>(
+      (data: DataMessage) => ({ data }),
+      this._pipeline.writer ?? failUndefined()
+    );
+
+    this.databaseBackend = new DatabaseBackendHost(feedWriter, this._snapshot?.database, {
+      snapshots: true // TODO(burdon): Config.
+    });
+    this._itemManager = new ItemManager(this._params.modelFactory);
+
+    // Connect pipeline to the database.
+    await this.databaseBackend.open(this._itemManager, this._params.modelFactory);
+
+    // Start message processing loop.
+    scheduleTask(this._ctx, async () => {
+      await this._consumePipeline();
+    });
+
+    this._createPeriodicSnapshots();
+  }
+
+  private _createPeriodicSnapshots() {
+    // Record last timeframe.
+    this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+      await this._saveLatestTimeframe();
+    });
+
+    this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
+      const latestTimeframe = this._pipeline?.state.timeframe;
+      if (!latestTimeframe) {
+        return;
+      }
+
+      // Save snapshot.
+      if (
+        latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() >
+        MESSAGES_PER_SNAPSHOT
+      ) {
+        const snapshot = await this._saveSnapshot();
+        this._lastAutomaticSnapshotTimeframe = snapshot.timeframe ?? failUndefined();
+        log('save', { snapshot });
+      }
+    });
+  }
+
+  private async _saveSnapshot() {
+    const snapshot = await this.createSnapshot();
+    const snapshotKey = await this._params.snapshotManager.store(snapshot);
+    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
+    return snapshot;
+  }
+
+  private async _saveLatestTimeframe() {
+    const latestTimeframe = this._pipeline?.state.timeframe;
+    if (latestTimeframe) {
+      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, latestTimeframe);
+    }
+  }
+
+  async close() {
+    try {
+      await this._saveLatestTimeframe();
+      await this._saveSnapshot();
+    } catch (err) {
+      log.catch(err);
+    }
+    await this._ctx.dispose();
+    await this._pipeline?.stop();
+    await this.databaseBackend?.close();
+    await this._itemManager?.destroy();
+    await this._params.snapshotManager.close();
+  }
+
+  createSnapshot(): SpaceSnapshot {
+    return {
+      spaceKey: this._params.spaceKey.asUint8Array(),
+      timeframe: this._pipeline?.state.timeframe ?? new Timeframe(),
+      database: this.databaseBackend!.createSnapshot()
+    };
+  }
+
+  private async _consumePipeline() {
+    assert(this._pipeline, 'Pipeline is not initialized.');
+    for await (const msg of this._pipeline.consume()) {
+      const { feedKey, seq, data } = msg;
+      log('processing message', { msg });
+
+      try {
+        if (data.payload.data) {
+          const feedInfo = this._params.feedInfoProvider(feedKey);
+          if (!feedInfo) {
+            log.error('Could not find feed.', { feedKey });
+            continue;
+          }
+
+          await this.databaseBackend!.echoProcessor({
+            data: data.payload.data.object,
+            meta: {
+              feedKey,
+              seq,
+              timeframe: data.timeframe,
+              memberKey: feedInfo.assertion.identityKey
+            }
+          });
+          this.onTimeframeReached.emit(data.timeframe);
+        }
+      } catch (err: any) {
+        log.catch(err);
+      }
+    }
+  }
+
+  async waitUntilTimeframe(timeframe: Timeframe) {
+    assert(this._pipeline, 'Pipeline is not initialized.');
+    await this._pipeline.state.waitUntilTimeframe(timeframe);
+  }
+}
+
+/**
+ * Increase all indexes by one so that we start processing the next mutation after the one in the snapshot.
+ */
+const snapshotTimeframeToStartingTimeframe = (snapshotTimeframe: Timeframe) => {
+  return snapshotTimeframe.map(([key, seq]) => [key, seq + 1]);
+};

package/src/space/genesis.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { CredentialGenerator } from '@dxos/credentials';
+import { failUndefined } from '@dxos/debug';
+import { Keyring } from '@dxos/keyring';
+import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
+
+import { Space } from './space';
+import { SigningContext } from './space-manager';
+
+export const spaceGenesis = async (keyring: Keyring, signingContext: SigningContext, space: Space) => {
+  // Write genesis credentials.
+  const generator = new CredentialGenerator(keyring, signingContext.identityKey, signingContext.deviceKey);
+
+  const credentials = [
+    ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey ?? failUndefined(), signingContext.profile)),
+    await generator.createFeedAdmission(space.key, space.dataFeedKey ?? failUndefined(), AdmittedFeed.Designation.DATA)
+  ];
+
+  for (const credential of credentials) {
+    await space.controlPipeline.writer.write({
+      credential: { credential }
+    });
+  }
+
+  return credentials;
+};

package/src/space/index.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './auth';
+export * from './space';
+export * from './space-manager';
+export * from './space-protocol';
+export * from './genesis';
+export * from './data-pipeline-controller';

package/src/space/replication.browser.test.ts

@@ -0,0 +1,64 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+// @dxos/test platform=browser
+
+import expect from 'expect';
+import waitForExpect from 'wait-for-expect';
+
+import { FeedFactory, FeedStore } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { createStorage } from '@dxos/random-access-storage';
+import { describe, test } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+
+import { valueEncoding } from '../common';
+
+describe('replication', () => {
+  test('replicates a feed through a direct stream', async () => {
+    // Some storage drivers may break when there are multiple storage instances.
+    const storage = createStorage();
+
+    // Creates an appropriate persistent storage for the browser: IDB in Chrome or File storage in Firefox.
+    const keyring1 = new Keyring();
+    const feedStore1 = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory('feeds1'),
+        signer: keyring1,
+        hypercore: {
+          valueEncoding
+        }
+      })
+    });
+
+    const keyring2 = new Keyring();
+    const feedStore2 = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory('feeds2'),
+        signer: keyring2,
+        hypercore: {
+          valueEncoding
+        }
+      })
+    });
+
+    const feed1 = await feedStore1.openFeed(await keyring1.createKey(), {
+      writable: true
+    });
+    const feed2 = await feedStore2.openFeed(feed1.key);
+
+    const stream1 = feed1.replicate(true, { live: true });
+    const stream2 = feed2.replicate(false, { live: true });
+    stream1.pipe(stream2).pipe(stream1);
+
+    await feed1.append({
+      timeframe: new Timeframe([[feed1.key, 123]])
+    });
+
+    await waitForExpect(() => {
+      expect(feed2.properties.length).toEqual(1);
+    });
+  });
+});

package/src/space/space-manager.browser.test.ts

@@ -0,0 +1,44 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @dxos/test platform=browser
+
+import { createStorage } from '@dxos/random-access-storage';
+import { describe, test, afterTest } from '@dxos/test';
+
+import { TestAgentBuilder, WebsocketNetworkManagerProvider } from '../testing';
+
+// TODO(burdon): Config.
+// Signal server will be started by the setup script.
+const SIGNAL_URL = 'ws://localhost:4000/.well-known/dx/signal';
+
+describe('space-manager', () => {
+  test.skip('invitations', async () => {
+    const builder = new TestAgentBuilder({
+      storage: createStorage(),
+      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL)
+    });
+    afterTest(async () => await builder.close());
+
+    const peer1 = await builder.createPeer();
+    const spaceManager1 = peer1.createSpaceManager();
+    await spaceManager1.open();
+
+    const peer2 = await builder.createPeer();
+    const spaceManager2 = peer2.createSpaceManager();
+    await spaceManager2.open();
+
+    afterTest(() => spaceManager1.close());
+    afterTest(() => spaceManager2.close());
+
+    // const space1 = await spaceManager1.createSpace();
+    // expect(space1.isOpen).to.be.true;
+
+    // TODO(burdon): Create invitation and join.
+    // TODO(burdon): Need to factor out DataInvitations from services.
+
+    // const space2 = await spaceManager2.acceptSpace();
+    // expect(space2.key).not.to.be.undefined;
+  });
+});