@dxos/echo-pipeline 0.1.41 → 0.1.45-next.45667d4

package/src/space/space.test.ts

@@ -4,10 +4,11 @@
 
 import expect from 'expect';
 import assert from 'node:assert';
+import { promisify } from 'node:util';
 
-import { CredentialGenerator } from '@dxos/credentials';
-import { AdmittedFeed } from '@dxos/protocols/proto/dxos/halo/credentials';
-import { describe, test, afterTest } from '@dxos/test';
+import { createCredential, CredentialGenerator } from '@dxos/credentials';
+import { log } from '@dxos/log';
+import { afterTest, describe, test } from '@dxos/test';
 
 import { TestAgentBuilder, testLocalDatabase } from '../testing';
 
@@ -19,32 +20,21 @@ describe('space/space', () => {
     const builder = new TestAgentBuilder();
     afterTest(async () => await builder.close());
     const agent = await builder.createPeer();
-    const [space, db] = await agent.createSpace();
+    const space = await agent.createSpace();
 
     await space.open();
+
     expect(space.isOpen).toBeTruthy();
     afterTest(() => space.close());
 
-    {
-      // Genesis
-      const generator = new CredentialGenerator(agent.keyring, agent.identityKey, agent.deviceKey);
-      const credentials = [
-        ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey!)),
-        await generator.createFeedAdmission(space.key, space.dataFeedKey!, AdmittedFeed.Designation.DATA)
-      ];
+    await agent.spaceGenesis(space);
 
-      for (const credential of credentials) {
-        await space.controlPipeline.writer.write({
-          credential: { credential }
-        });
-      }
-
-      // TODO(burdon): Debugging only.
-      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
-    }
+    await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+    await space.initializeDataPipeline();
+    await space.dataPipeline.ensureEpochInitialized();
 
-    assert(db.databaseBackend);
-    await testLocalDatabase(db);
+    assert(space.dataPipeline.databaseHost);
+    await testLocalDatabase(space.dataPipeline);
 
     await builder.close();
     expect(space.isOpen).toBeFalsy();
@@ -57,47 +47,36 @@ describe('space/space', () => {
     //
     // Agent 1
     //
-    const [agent1, space1, db1] = await run(async () => {
+    const [agent1, space1] = await run(async () => {
       const agent = await builder.createPeer();
-      const [space, db] = await agent.createSpace(agent.identityKey);
+      const space = await agent.createSpace(agent.identityKey);
 
       await space.open();
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
-      {
-        // Genesis
-        const generator = new CredentialGenerator(agent.keyring, agent.identityKey, agent.deviceKey);
-        const credentials = [
-          ...(await generator.createSpaceGenesis(space.key, space.controlFeedKey!)),
-          await generator.createFeedAdmission(space.key, space.dataFeedKey!, AdmittedFeed.Designation.DATA)
-        ];
-
-        for (const credential of credentials) {
-          await space.controlPipeline.writer.write({
-            credential: { credential }
-          });
-        }
-
-        await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
-      }
+      await agent.spaceGenesis(space);
+
+      await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);
+      await space.initializeDataPipeline();
+      await space.dataPipeline.ensureEpochInitialized();
 
-      return [agent, space, db];
+      return [agent, space];
     });
 
     //
     // Agent 2
     //
-    const [agent2, space2, db2] = await run(async () => {
+    const [agent2, space2] = await run(async () => {
       // NOTE: The genesisKey would be passed as part of the invitation.
       const agent = await builder.createPeer();
-      const [space, db] = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey);
+      const space = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey);
 
       await space.open();
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
-      return [agent, space, db];
+      return [agent, space];
     });
 
     expect(agent1).toBeDefined();
@@ -112,16 +91,19 @@ describe('space/space', () => {
         agent2.deviceKey,
         space2.controlFeedKey!,
         space2.dataFeedKey!,
-        space1.genesisFeedKey
+        space1.genesisFeedKey,
       );
 
       for (const credential of credentials) {
         await space1.controlPipeline.writer.write({
-          credential: { credential }
+          credential: { credential },
         });
       }
     }
 
+    await space2.initializeDataPipeline();
+    await space2.dataPipeline.ensureEpochInitialized();
+
     {
       // Initial data exchange.
 
@@ -134,11 +116,118 @@ describe('space/space', () => {
 
     // TODO(burdon): Write multiple items (extract for all tests).
 
-    await testLocalDatabase(db1, db2);
-    await testLocalDatabase(db2, db1);
+    await testLocalDatabase(space1.dataPipeline, space2.dataPipeline);
+    await testLocalDatabase(space2.dataPipeline, space1.dataPipeline);
 
     await builder.close();
     expect(space1.isOpen).toBeFalsy();
     expect(space2.isOpen).toBeFalsy();
   });
+
+  test('open & close', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const agent = await builder.createPeer();
+    const space1 = await agent.createSpace();
+
+    await space1.open();
+
+    expect(space1.isOpen).toBeTruthy();
+    afterTest(() => space1.close());
+
+    await agent.spaceGenesis(space1);
+
+    await space1.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+    await space1.initializeDataPipeline();
+    await space1.dataPipeline.ensureEpochInitialized();
+
+    assert(space1.dataPipeline.databaseHost);
+    await testLocalDatabase(space1.dataPipeline);
+
+    const objectCount = space1.dataPipeline.itemManager.entities.size;
+
+    await space1.close();
+    expect(space1.isOpen).toBeFalsy();
+
+    // Re-open.
+    const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
+
+    await space2.open();
+
+    await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
+    await space2.initializeDataPipeline();
+    await space2.dataPipeline.ensureEpochInitialized();
+
+    space2.dataPipeline.setTargetTimeframe(space2.dataPipeline.pipelineState!.endTimeframe);
+    await space2.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
+
+    assert(space2.dataPipeline.databaseHost);
+    expect(space2.dataPipeline.itemManager.entities.size).toEqual(objectCount);
+
+    await testLocalDatabase(space2.dataPipeline);
+  });
+
+  test('create epoch', async () => {
+    const builder = new TestAgentBuilder();
+    afterTest(async () => await builder.close());
+    const agent = await builder.createPeer();
+    const space1 = await agent.createSpace();
+
+    await space1.open();
+
+    expect(space1.isOpen).toBeTruthy();
+    afterTest(() => space1.close());
+
+    await agent.spaceGenesis(space1);
+
+    await space1.controlPipeline.state!.waitUntilTimeframe(space1.controlPipeline.state!.endTimeframe);
+    await space1.initializeDataPipeline();
+    await space1.dataPipeline.ensureEpochInitialized();
+
+    assert(space1.dataPipeline.databaseHost);
+    await testLocalDatabase(space1.dataPipeline);
+
+    const objectCount = space1.dataPipeline.itemManager.entities.size;
+
+    const epoch = await space1.dataPipeline.createEpoch();
+    await space1.controlPipeline.writer.write({
+      credential: {
+        credential: await createCredential({
+          issuer: agent.identityKey,
+          subject: space1.key,
+          assertion: {
+            '@type': 'dxos.halo.credentials.Epoch',
+            ...epoch,
+          },
+          signer: agent.keyring,
+        }),
+      },
+    });
+
+    await space1.close();
+    expect(space1.isOpen).toBeFalsy();
+
+    // Clear the data feed - epoch snapshot should have the data.
+    const feed = await agent.feedStore.openFeed(space1.dataFeedKey!);
+    await promisify(feed.core.clear.bind(feed.core))(0, feed.length);
+
+    log.break();
+
+    // Re-open.
+    const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
+
+    await space2.open();
+
+    await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
+    await space2.initializeDataPipeline();
+    await space2.dataPipeline.ensureEpochInitialized();
+
+    space2.dataPipeline.setTargetTimeframe(space2.dataPipeline.pipelineState!.endTimeframe);
+    await space2.dataPipeline.pipelineState!.waitUntilReachedTargetTimeframe();
+
+    assert(space2.dataPipeline.databaseHost);
+    expect(space2.dataPipeline.itemManager.entities.size).toEqual(objectCount);
+
+    await testLocalDatabase(space2.dataPipeline);
+  });
 });

package/src/space/space.ts

@@ -5,17 +5,21 @@
 import assert from 'node:assert';
 
 import { Event, synchronized, trackLeaks, Lock } from '@dxos/async';
-import { FeedInfo } from '@dxos/credentials';
+import { CredentialConsumer, FeedInfo } from '@dxos/credentials';
 import { FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
+import { ModelFactory } from '@dxos/model-factory';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { AdmittedFeed, Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
 import { AsyncCallback, Callback } from '@dxos/util';
 
-import { Pipeline, PipelineAccessor } from '../pipeline';
+import { SnapshotManager } from '../dbhost';
+import { MetadataStore } from '../metadata';
+import { PipelineAccessor } from '../pipeline';
 import { ControlPipeline } from './control-pipeline';
+import { DataPipeline } from './data-pipeline';
 import { SpaceProtocol } from './space-protocol';
 
 // TODO(burdon): Factor out?
@@ -26,6 +30,13 @@ export type SpaceParams = {
   protocol: SpaceProtocol;
   genesisFeed: FeedWrapper<FeedMessage>;
   feedProvider: FeedProvider;
+  modelFactory: ModelFactory;
+  metadataStore: MetadataStore;
+  snapshotManager: SnapshotManager;
+  memberKey: PublicKey;
+
+  // TODO(dmaretskyi): Superseded by epochs.
+  snapshotId?: string | undefined;
 };
 
 export type CreatePipelineParams = {
@@ -49,36 +60,44 @@ export class Space {
   private readonly _genesisFeedKey: PublicKey;
   private readonly _feedProvider: FeedProvider;
   private readonly _controlPipeline: ControlPipeline;
+  private readonly _dataPipeline: DataPipeline;
+  private readonly _snapshotManager: SnapshotManager;
+
+  // Processes epoch credentials.
+  private _dataPipelineCredentialConsumer?: CredentialConsumer<any> = undefined;
 
   private _isOpen = false;
   private _controlFeed?: FeedWrapper<FeedMessage>;
   private _dataFeed?: FeedWrapper<FeedMessage>;
-  private _dataPipeline?: Pipeline;
 
-  constructor({ spaceKey, protocol, genesisFeed, feedProvider }: SpaceParams) {
-    assert(spaceKey && feedProvider);
-    this._key = spaceKey;
-    this._genesisFeedKey = genesisFeed.key;
-    this._feedProvider = feedProvider;
-
-    // TODO(dmaretskyi): Maybe reuse createPipeline method.
-    this._controlPipeline = new ControlPipeline({ spaceKey, genesisFeed, feedProvider });
+  constructor(params: SpaceParams) {
+    assert(params.spaceKey && params.feedProvider);
+    this._key = params.spaceKey;
+    this._genesisFeedKey = params.genesisFeed.key;
+    this._feedProvider = params.feedProvider;
+    this._snapshotManager = params.snapshotManager;
+
+    this._controlPipeline = new ControlPipeline({
+      spaceKey: params.spaceKey,
+      genesisFeed: params.genesisFeed,
+      feedProvider: params.feedProvider,
+    });
 
     // TODO(dmaretskyi): Feed set abstraction.
     this._controlPipeline.onFeedAdmitted.set(async (info) => {
       if (info.assertion.designation === AdmittedFeed.Designation.DATA) {
         // We will add all existing data feeds when the data pipeline is initialized.
         await this._addFeedLock.executeSynchronized(async () => {
-          if (this._dataPipeline) {
-            if (!this._dataPipeline.hasFeed(info.key)) {
-              return this._dataPipeline.addFeed(await this._feedProvider(info.key));
+          if (this._dataPipeline.pipeline) {
+            if (!this._dataPipeline.pipeline.hasFeed(info.key)) {
+              return this._dataPipeline.pipeline.addFeed(await this._feedProvider(info.key));
             }
           }
         });
       }
 
-      if (!info.key.equals(genesisFeed.key)) {
-        this.protocol.addFeed(await feedProvider(info.key));
+      if (!info.key.equals(params.genesisFeed.key)) {
+        this.protocol.addFeed(await params.feedProvider(info.key));
       }
     });
 
@@ -89,8 +108,32 @@ export class Space {
     });
 
     // Start replicating the genesis feed.
-    this.protocol = protocol;
-    this.protocol.addFeed(genesisFeed);
+    this.protocol = params.protocol;
+    this.protocol.addFeed(params.genesisFeed);
+
+    this._dataPipeline = new DataPipeline({
+      modelFactory: params.modelFactory,
+      metadataStore: params.metadataStore,
+      snapshotManager: params.snapshotManager,
+      memberKey: params.memberKey,
+      spaceKey: this._key,
+      feedInfoProvider: (feedKey) => this._controlPipeline.spaceState.feeds.get(feedKey),
+      snapshotId: params.snapshotId,
+      onPipelineCreated: async (pipeline) => {
+        if (this._dataFeed) {
+          pipeline.setWriteFeed(this._dataFeed);
+        }
+
+        // Add existing feeds.
+        await this._addFeedLock.executeSynchronized(async () => {
+          for (const feed of this._controlPipeline.spaceState.feeds.values()) {
+            if (feed.assertion.designation === AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
+              await pipeline.addFeed(await this._feedProvider(feed.key));
+            }
+          }
+        });
+      },
+    });
   }
 
   @logInfo
@@ -125,6 +168,14 @@ export class Space {
     return this._controlPipeline.pipeline;
   }
 
+  get dataPipeline(): DataPipeline {
+    return this._dataPipeline;
+  }
+
+  get snapshotManager(): SnapshotManager {
+    return this._snapshotManager;
+  }
+
   setControlFeed(feed: FeedWrapper<FeedMessage>) {
     assert(!this._controlFeed, 'Control feed already set.');
     this._controlFeed = feed;
@@ -135,7 +186,7 @@ export class Space {
   setDataFeed(feed: FeedWrapper<FeedMessage>) {
     assert(!this._dataFeed, 'Data feed already set.');
     this._dataFeed = feed;
-    this._dataPipeline?.setWriteFeed(feed);
+    this._dataPipeline.pipeline?.setWriteFeed(feed);
     return this;
   }
 
@@ -163,6 +214,10 @@ export class Space {
     await this._controlPipeline.start();
     await this.protocol.start();
 
+    this._dataPipelineCredentialConsumer = this._controlPipeline.spaceState.registerProcessor(
+      this._dataPipeline.createCredentialProcessor(),
+    );
+
     this._isOpen = true;
     log('opened');
   }
@@ -173,6 +228,10 @@ export class Space {
     if (!this._isOpen) {
       return;
     }
+    await this._dataPipelineCredentialConsumer?.close();
+    this._dataPipelineCredentialConsumer = undefined;
+
+    await this._dataPipeline.close();
 
     // Closes in reverse order to open.
     await this.protocol.stop();
@@ -182,27 +241,10 @@ export class Space {
     log('closed');
   }
 
-  // TODO(dmaretskyi): Make reusable.
-  async createDataPipeline({ start }: CreatePipelineParams) {
+  async initializeDataPipeline() {
+    log.info('initializeDataPipeline');
     assert(this._isOpen, 'Space must be open to initialize data pipeline.');
-    assert(!this._dataPipeline, 'Data pipeline already initialized.');
-
-    const pipeline = new Pipeline(start);
-    if (this._dataFeed) {
-      pipeline.setWriteFeed(this._dataFeed);
-    }
-
-    this._dataPipeline = pipeline;
-
-    // Add existing feeds.
-    await this._addFeedLock.executeSynchronized(async () => {
-      for (const feed of this._controlPipeline.spaceState.feeds.values()) {
-        if (feed.assertion.designation === AdmittedFeed.Designation.DATA && !pipeline.hasFeed(feed.key)) {
-          await pipeline.addFeed(await this._feedProvider(feed.key));
-        }
-      }
-    });
-
-    return pipeline;
+    await this._dataPipeline.open();
+    await this._dataPipelineCredentialConsumer!.open();
   }
 }

package/src/testing/database-test-rig.ts

@@ -64,27 +64,28 @@ export class DatabaseTestPeer {
 
   async open() {
     this.hostItems = new ItemManager(this.modelFactory);
-    this.host = new DatabaseHost(
-      {
-        write: async (message) => {
-          const seq =
-            this.feedMessages.push({
-              timeframe: this.timeframe,
-              payload: {
-                data: message
-              }
-            }) - 1;
-
-          await this._onConfirm.waitFor(() => this.confirmed >= seq);
-          return {
-            seq,
-            feedKey: this.key
-          };
-        }
+    this.host = new DatabaseHost({
+      write: async (message) => {
+        const seq =
+          this.feedMessages.push({
+            timeframe: this.timeframe,
+            payload: {
+              data: message,
+            },
+          }) - 1;
+
+        await this._onConfirm.waitFor(() => this.confirmed >= seq);
+        return {
+          seq,
+          feedKey: this.key,
+        };
       },
-      this.snapshot?.database
-    );
+    });
+
     await this.host.open(this.hostItems, this.modelFactory);
+    if (this.snapshot) {
+      this.host._itemDemuxer.restoreFromSnapshot(this.snapshot.database);
+    }
 
     this.proxy = new DatabaseProxy(this.host.createDataServiceHost(), SPACE_KEY);
     this.items = new ItemManager(this.modelFactory);
@@ -109,7 +110,7 @@ export class DatabaseTestPeer {
   replicate(to?: Timeframe) {
     const toTimeframe = Timeframe.merge(
       to ?? new Timeframe(Array.from(this.rig.peers.values()).map((peer) => [peer.key, peer.confirmed])),
-      this.timeframe
+      this.timeframe,
     );
     toTimeframe.set(this.key, this.confirmed);
 
@@ -133,7 +134,7 @@ export class DatabaseTestPeer {
     this.snapshot = {
       spaceKey: SPACE_KEY.asUint8Array(),
       database: this.host.createSnapshot(),
-      timeframe: this.timeframe
+      timeframe: this.timeframe,
     };
     return this.snapshot;
   }
@@ -151,7 +152,7 @@ export class DatabaseTestPeer {
           message && {
             feedKey: peer.key,
             seq: seq + 1,
-            data: message
+            data: message,
           }
         );
       })
@@ -186,8 +187,8 @@ export class DatabaseTestPeer {
             feedKey: candidate.feedKey,
             seq: candidate.seq,
             memberKey: candidate.feedKey,
-            timeframe: candidate.data.timeframe
-          }
+            timeframe: candidate.data.timeframe,
+          },
         });
         this.timeframe = Timeframe.merge(this.timeframe, new Timeframe([[candidate.feedKey, candidate.seq]]));
       }