@dxos/echo-pipeline 0.1.41 → 0.1.45-next.45667d4

package/src/space/data-pipeline.ts

@@ -4,19 +4,21 @@
 
 import assert from 'node:assert';
 
-import { scheduleTask, synchronized, trackLeaks } from '@dxos/async';
+import { Event, scheduleTask, synchronized, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
-import { FeedInfo } from '@dxos/credentials';
-import { failUndefined } from '@dxos/debug';
-import { ItemManager } from '@dxos/echo-db';
+import { CredentialProcessor, FeedInfo, getCredentialAssertion } from '@dxos/credentials';
+import { getStateMachineFromItem, ItemManager } from '@dxos/echo-db';
+import { FeedWriter } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { ModelFactory } from '@dxos/model-factory';
-import { DataMessage, FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { ObjectSnapshot } from '@dxos/protocols/proto/dxos/echo/model/document';
 import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { Credential, Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
 
-import { createMappedFeedWriter } from '../common';
 import { DatabaseHost, SnapshotManager } from '../dbhost';
 import { MetadataStore } from '../metadata';
 import { Pipeline } from '../pipeline';
@@ -33,6 +35,11 @@ export type DataPipelineParams = {
   spaceKey: PublicKey;
   feedInfoProvider: (feedKey: PublicKey) => FeedInfo | undefined;
   snapshotId: string | undefined;
+
+  /**
+   * Called once.
+   */
+  onPipelineCreated: (pipeline: Pipeline) => Promise<void>;
 };
 
 /**
@@ -50,11 +57,6 @@ const AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5000;
  */
 const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 
-/**
- * Flag to disable automatic local snapshots.
- */
-const DISABLE_SNAPSHOT_CACHE = true;
-
 /**
  * Controls data pipeline in the space.
  * Consumes the pipeline and updates the database.
@@ -63,9 +65,7 @@ const DISABLE_SNAPSHOT_CACHE = true;
 @trackLeaks('open', 'close')
 export class DataPipeline {
   private _ctx = new Context();
-  private _spaceContext!: PipelineFactory;
   private _pipeline?: Pipeline;
-  private _snapshot?: SpaceSnapshot;
   private _targetTimeframe?: Timeframe;
 
   private _lastAutomaticSnapshotTimeframe = new Timeframe();
@@ -76,23 +76,22 @@ export class DataPipeline {
 
   constructor(private readonly _params: DataPipelineParams) {}
 
-  public _itemManager!: ItemManager;
-  public databaseBackend?: DatabaseHost;
+  public itemManager!: ItemManager;
+  public databaseHost?: DatabaseHost;
+
+  public currentEpoch?: Credential;
+  public onNewEpoch = new Event<Credential>();
 
   get isOpen() {
     return this._isOpen;
   }
 
-  get pipelineState() {
-    return this._pipeline?.state;
-  }
-
-  get snapshotTimeframe() {
-    return this._snapshot?.timeframe;
+  get pipeline() {
+    return this._pipeline;
   }
 
-  getStartTimeframe(): Timeframe {
-    return snapshotTimeframeToStartingTimeframe(this.snapshotTimeframe ?? new Timeframe());
+  get pipelineState() {
+    return this._pipeline?.state;
   }
 
   setTargetTimeframe(timeframe: Timeframe) {
@@ -100,34 +99,52 @@ export class DataPipeline {
     this._pipeline?.state.setTargetTimeframe(timeframe);
   }
 
+  createCredentialProcessor(): CredentialProcessor {
+    return {
+      process: async (credential) => {
+        const assertion = getCredentialAssertion(credential);
+        if (assertion['@type'] !== 'dxos.halo.credentials.Epoch') {
+          return;
+        }
+
+        log('new epoch', { credential });
+        await this._processEpoch(assertion);
+
+        this.currentEpoch = credential;
+        this.onNewEpoch.emit(credential);
+      },
+    };
+  }
+
   @synchronized
-  async open(spaceContext: PipelineFactory) {
+  async open() {
     if (this._isOpen) {
       return;
     }
 
-    this._spaceContext = spaceContext;
-    if (this._params.snapshotId && !DISABLE_SNAPSHOT_CACHE) {
-      this._snapshot = await this._params.snapshotManager.load(this._params.snapshotId);
-      this._lastAutomaticSnapshotTimeframe = this._snapshot?.timeframe ?? new Timeframe();
-    }
+    this._pipeline = new Pipeline();
+    await this._params.onPipelineCreated(this._pipeline);
+    await this._pipeline.start();
+    await this._pipeline.pause(); // Start paused until we have the first epoch.
 
-    this._pipeline = await this._spaceContext.openPipeline(this.getStartTimeframe());
     if (this._targetTimeframe) {
       this._pipeline.state.setTargetTimeframe(this._targetTimeframe);
     }
 
     // Create database backend.
-    const feedWriter = createMappedFeedWriter<DataMessage, FeedMessage.Payload>(
-      (data: DataMessage) => ({ data }),
-      this._pipeline.writer ?? failUndefined()
-    );
+    const feedWriter: FeedWriter<DataMessage> = {
+      write: (data, options) => {
+        assert(this._pipeline, 'Pipeline is not initialized.');
+        assert(this.currentEpoch, 'Epoch is not initialized.');
+        return this._pipeline.writer.write({ data }, options);
+      },
+    };
 
-    this.databaseBackend = new DatabaseHost(feedWriter, this._snapshot?.database);
-    this._itemManager = new ItemManager(this._params.modelFactory);
+    this.databaseHost = new DatabaseHost(feedWriter);
+    this.itemManager = new ItemManager(this._params.modelFactory);
 
     // Connect pipeline to the database.
-    await this.databaseBackend.open(this._itemManager, this._params.modelFactory);
+    await this.databaseHost.open(this.itemManager, this._params.modelFactory);
 
     // Start message processing loop.
     scheduleTask(this._ctx, async () => {
@@ -150,18 +167,16 @@ export class DataPipeline {
 
     // NOTE: Make sure the processing is stopped BEFORE we save the snapshot.
     try {
+      await this._saveCache();
       if (this._pipeline) {
         await this._saveTargetTimeframe(this._pipeline.state.timeframe);
-        if (!DISABLE_SNAPSHOT_CACHE) {
-          await this._saveSnapshot(this._pipeline.state.timeframe);
-        }
       }
     } catch (err) {
       log.catch(err);
     }
 
-    await this.databaseBackend?.close();
-    await this._itemManager?.destroy();
+    await this.databaseHost?.close();
+    await this.itemManager?.destroy();
     await this._params.snapshotManager.close();
   }
 
@@ -169,7 +184,7 @@ export class DataPipeline {
     assert(this._pipeline, 'Pipeline is not initialized.');
     for await (const msg of this._pipeline.consume()) {
       const { feedKey, seq, data } = msg;
-      log('processing message', { msg });
+      log('processing message', { feedKey, seq });
 
       try {
         if (data.payload.data) {
@@ -179,14 +194,14 @@ export class DataPipeline {
             continue;
           }
 
-          await this.databaseBackend!.echoProcessor({
+          await this.databaseHost!.echoProcessor({
             batch: data.payload.data.batch,
             meta: {
               feedKey,
               seq,
               timeframe: data.timeframe,
-              memberKey: feedInfo.assertion.identityKey
-            }
+              memberKey: feedInfo.assertion.identityKey,
+            },
           });
 
           // Timeframe clock is not updated yet
@@ -198,29 +213,45 @@ export class DataPipeline {
     }
   }
 
-  private _createSnapshot(timeframe: Timeframe): SpaceSnapshot {
-    assert(this.databaseBackend, 'Database backend is not initialized.');
+  private _createSnapshot(): SpaceSnapshot {
+    assert(this.databaseHost, 'Database backend is not initialized.');
     return {
       spaceKey: this._params.spaceKey.asUint8Array(),
-      timeframe,
-      database: this.databaseBackend!.createSnapshot()
+      timeframe: this._pipeline!.state.timeframe,
+      database: this.databaseHost!.createSnapshot(),
     };
   }
 
-  private async _saveSnapshot(timeframe: Timeframe) {
-    const snapshot = await this._createSnapshot(timeframe);
-    const snapshotKey = await this._params.snapshotManager.store(snapshot);
-    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
-    return snapshot;
-  }
-
   private async _saveTargetTimeframe(timeframe: Timeframe) {
     const newTimeframe = Timeframe.merge(this._targetTimeframe ?? new Timeframe(), timeframe);
     await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
     this._targetTimeframe = newTimeframe;
   }
 
+  private async _saveCache() {
+    const cache: SpaceCache = {};
+
+    try {
+      // Add properties to cache.
+      const propertiesItem = this.itemManager.items.find(
+        (item) =>
+          item.modelMeta?.type === 'dxos:model/document' &&
+          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === 'dxos.sdk.client.Properties',
+      );
+      if (propertiesItem) {
+        cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot() as ObjectSnapshot;
+      }
+    } catch (err) {
+      log.warn('Failed to cache properties', err);
+    }
+
+    // Save cache.
+    await this._params.metadataStore.setCache(this._params.spaceKey, cache);
+  }
+
   private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
+    // TODO(dmaretskyi): Replace this with a proper debounce/throttle.
+
     if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
       this._lastTimeframeSaveTime = Date.now();
 
@@ -228,27 +259,58 @@ export class DataPipeline {
     }
 
     if (
-      !DISABLE_SNAPSHOT_CACHE &&
       Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL &&
       timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT
     ) {
-      this._lastSnapshotSaveTime = Date.now();
+      await this._saveCache();
+    }
+  }
 
-      const snapshot = await this._saveSnapshot(timeframe);
-      this._lastAutomaticSnapshotTimeframe = snapshot.timeframe ?? failUndefined();
-      log('save', { snapshot });
+  @synchronized
+  private async _processEpoch(epoch: Epoch) {
+    assert(this._isOpen); // TODO: In the future we might process epochs before we are open so that data pipeline starts from the last one.
+    assert(this._pipeline);
+
+    if (epoch.snapshotCid) {
+      const snapshot = await this._params.snapshotManager.load(epoch.snapshotCid);
+
+      // TODO(dmaretskyi): Clearing old items + events.
+      this.databaseHost!._itemDemuxer.restoreFromSnapshot(snapshot.database);
     }
+
+    await this._pipeline.pause();
+    await this._pipeline.setCursor(epoch.timeframe);
+    await this._pipeline.unpause();
   }
 
   async waitUntilTimeframe(timeframe: Timeframe) {
     assert(this._pipeline, 'Pipeline is not initialized.');
     await this._pipeline.state.waitUntilTimeframe(timeframe);
   }
-}
 
-/**
- * Increase all indexes by one so that we start processing the next mutation after the one in the snapshot.
- */
-const snapshotTimeframeToStartingTimeframe = (snapshotTimeframe: Timeframe) => {
-  return snapshotTimeframe.map(([key, seq]) => [key, seq + 1]);
-};
+  @synchronized
+  async createEpoch(): Promise<Epoch> {
+    assert(this._pipeline);
+    assert(this.currentEpoch);
+
+    await this._pipeline.pause();
+
+    const snapshot = await this._createSnapshot();
+    const snapshotCid = await this._params.snapshotManager.store(snapshot);
+
+    const epoch: Epoch = {
+      previousId: this.currentEpoch.id,
+      timeframe: this._pipeline.state.timeframe,
+      number: (this.currentEpoch.subject.assertion as Epoch).number + 1,
+      snapshotCid,
+    };
+
+    await this._pipeline.unpause();
+
+    return epoch;
+  }
+
+  async ensureEpochInitialized() {
+    await this.onNewEpoch.waitForCondition(() => !!this.currentEpoch);
+  }
+}

package/src/space/index.ts

@@ -6,5 +6,4 @@ export * from './auth';
 export * from './space';
 export * from './space-manager';
 export * from './space-protocol';
-export * from './genesis';
 export * from './data-pipeline';

package/src/space/replication.browser.test.ts

@@ -28,9 +28,9 @@ describe('replication', () => {
         root: storage.createDirectory('feeds1'),
         signer: keyring1,
         hypercore: {
-          valueEncoding
-        }
-      })
+          valueEncoding,
+        },
+      }),
     });
 
     const keyring2 = new Keyring();
@@ -39,13 +39,13 @@ describe('replication', () => {
         root: storage.createDirectory('feeds2'),
         signer: keyring2,
         hypercore: {
-          valueEncoding
-        }
-      })
+          valueEncoding,
+        },
+      }),
     });
 
     const feed1 = await feedStore1.openFeed(await keyring1.createKey(), {
-      writable: true
+      writable: true,
     });
     const feed2 = await feedStore2.openFeed(feed1.key);
 
@@ -54,7 +54,7 @@ describe('replication', () => {
     stream1.pipe(stream2).pipe(stream1);
 
     await feed1.append({
-      timeframe: new Timeframe([[feed1.key, 123]])
+      timeframe: new Timeframe([[feed1.key, 123]]),
     });
 
     await waitForExpect(() => {

package/src/space/space-manager.browser.test.ts

@@ -17,16 +17,16 @@ describe('space-manager', () => {
   test.skip('invitations', async () => {
     const builder = new TestAgentBuilder({
       storage: createStorage(),
-      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL)
+      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL),
     });
     afterTest(async () => await builder.close());
 
     const peer1 = await builder.createPeer();
-    const spaceManager1 = peer1.createSpaceManager();
+    const spaceManager1 = peer1.spaceManager;
     await spaceManager1.open();
 
     const peer2 = await builder.createPeer();
-    const spaceManager2 = peer2.createSpaceManager();
+    const spaceManager2 = peer2.spaceManager;
     await spaceManager2.open();
 
     afterTest(() => spaceManager1.close());

package/src/space/space-manager.ts

@@ -3,39 +3,35 @@
 //
 
 import { synchronized, trackLeaks } from '@dxos/async';
-import { CredentialSigner } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
+import { ModelFactory } from '@dxos/model-factory';
 import { NetworkManager } from '@dxos/network-manager';
 import { trace } from '@dxos/protocols';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
-import { Credential, ProfileDocument } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Teleport } from '@dxos/teleport';
 import { ComplexMap } from '@dxos/util';
 
+import { SnapshotManager, SnapshotStore } from '../dbhost';
+import { MetadataStore } from '../metadata';
 import { Space } from './space';
 import { SpaceProtocol, SwarmIdentity } from './space-protocol';
 
-// TODO(burdon): Factor out to DataSpaceManager
-export interface SigningContext {
-  identityKey: PublicKey;
-  deviceKey: PublicKey;
-  credentialSigner: CredentialSigner; // TODO(burdon): Already has keyring.
-  recordCredential: (credential: Credential) => Promise<void>;
-  profile?: ProfileDocument;
-}
-
 export type SpaceManagerParams = {
   feedStore: FeedStore<FeedMessage>;
   networkManager: NetworkManager;
+  modelFactory: ModelFactory;
+  metadataStore: MetadataStore;
+  snapshotStore: SnapshotStore;
 };
 
 export type ConstructSpaceParams = {
   metadata: SpaceMetadata;
   swarmIdentity: SwarmIdentity;
+  memberKey: PublicKey;
   onNetworkConnection: (session: Teleport) => void;
 };
 
@@ -47,13 +43,18 @@ export class SpaceManager {
   private readonly _spaces = new ComplexMap<PublicKey, Space>(PublicKey.hash);
   private readonly _feedStore: FeedStore<FeedMessage>;
   private readonly _networkManager: NetworkManager;
+  private readonly _modelFactory: ModelFactory;
+  private readonly _metadataStore: MetadataStore;
+  private readonly _snapshotStore: SnapshotStore;
   private readonly _instanceId = PublicKey.random().toHex();
-  public _traceParent?: string;
 
-  constructor({ feedStore, networkManager }: SpaceManagerParams) {
+  constructor({ feedStore, networkManager, modelFactory, metadataStore, snapshotStore }: SpaceManagerParams) {
     // TODO(burdon): Assert.
     this._feedStore = feedStore;
     this._networkManager = networkManager;
+    this._modelFactory = modelFactory;
+    this._metadataStore = metadataStore;
+    this._snapshotStore = snapshotStore;
   }
 
   // TODO(burdon): Remove.
@@ -62,17 +63,15 @@ export class SpaceManager {
   }
 
   @synchronized
-  async open() {
-    log.trace('dxos.echo.space-manager', trace.begin({ id: this._instanceId, parentId: this._traceParent }));
-  }
+  async open() {}
 
   @synchronized
   async close() {
     await Promise.all([...this._spaces.values()].map((space) => space.close()));
-    log.trace('dxos.echo.space-manager', trace.end({ id: this._instanceId }));
   }
 
-  async constructSpace({ metadata, swarmIdentity, onNetworkConnection }: ConstructSpaceParams) {
+  async constructSpace({ metadata, swarmIdentity, onNetworkConnection, memberKey }: ConstructSpaceParams) {
+    log.trace('dxos.echo.space-manager.construct-space', trace.begin({ id: this._instanceId }));
     log('constructing space...', { spaceKey: metadata.genesisFeedKey });
 
     // The genesis feed will be the same as the control feed if the space was created by the local agent.
@@ -83,16 +82,22 @@ export class SpaceManager {
       topic: spaceKey,
       swarmIdentity,
       networkManager: this._networkManager,
-      onSessionAuth: onNetworkConnection
+      onSessionAuth: onNetworkConnection,
     });
+    const snapshotManager = new SnapshotManager(this._snapshotStore);
 
     const space = new Space({
       spaceKey,
       protocol,
       genesisFeed,
-      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey)
+      feedProvider: (feedKey) => this._feedStore.openFeed(feedKey),
+      modelFactory: this._modelFactory,
+      metadataStore: this._metadataStore,
+      snapshotManager,
+      memberKey,
     });
     this._spaces.set(space.key, space);
+    log.trace('dxos.echo.space-manager.construct-space', trace.end({ id: this._instanceId }));
     return space;
   }
 }

package/src/space/space-protocol.browser.test.ts

@@ -93,7 +93,7 @@ describe('space/space-protocol', () => {
   test('replicates a feed through a webrtc connection', async () => {
     const builder = new TestAgentBuilder({
       storage: createStorage(),
-      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL)
+      networkManagerProvider: WebsocketNetworkManagerProvider(SIGNAL_URL),
     });
     afterTest(async () => await builder.close());
 

package/src/space/space-protocol.test.ts

@@ -51,12 +51,12 @@ describe('space/space-protocol', () => {
       swarmIdentity: {
         peerKey: peerId1,
         credentialProvider: MOCK_AUTH_PROVIDER,
-        credentialAuthenticator: async () => false // Reject everyone.
+        credentialAuthenticator: async () => false, // Reject everyone.
       },
       networkManager: new NetworkManager({
         signalManager: new MemorySignalManager(signalContext),
-        transportFactory: MemoryTransportFactory
-      })
+        transportFactory: MemoryTransportFactory,
+      }),
     });
 
     const protocol2 = new SpaceProtocol({
@@ -64,12 +64,12 @@ describe('space/space-protocol', () => {
       swarmIdentity: {
         peerKey: peerId2,
         credentialProvider: MOCK_AUTH_PROVIDER,
-        credentialAuthenticator: MOCK_AUTH_VERIFIER
+        credentialAuthenticator: MOCK_AUTH_VERIFIER,
       },
       networkManager: new NetworkManager({
         signalManager: new MemorySignalManager(signalContext),
-        transportFactory: MemoryTransportFactory
-      })
+        transportFactory: MemoryTransportFactory,
+      }),
     });
 
     await protocol1.start();

package/src/space/space-protocol.ts

@@ -12,7 +12,7 @@ import {
   SwarmConnection,
   WireProtocol,
   WireProtocolParams,
-  WireProtocolProvider
+  WireProtocolProvider,
 } from '@dxos/network-manager';
 import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Teleport } from '@dxos/teleport';
@@ -101,7 +101,7 @@ export class SpaceProtocol {
     const topologyConfig = {
       originateConnections: 4,
       maxPeers: 10,
-      sampleSize: 20
+      sampleSize: 20,
     };
 
     log('starting...');
@@ -110,7 +110,7 @@ export class SpaceProtocol {
       peerId: this._swarmIdentity.peerKey,
       topic: this._topic,
       topology: new MMSTTopology(topologyConfig),
-      label: `Protocol swarm: ${this._topic}`
+      label: `Protocol swarm: ${this._topic}`,
     });
 
     log('started');
@@ -129,7 +129,7 @@ export class SpaceProtocol {
       const session = new SpaceProtocolSession({
         wireParams,
         swarmIdentity: this._swarmIdentity,
-        onSessionAuth: this._onSessionAuth
+        onSessionAuth: this._onSessionAuth,
       });
       this._sessions.set(wireParams.remotePeerId, session);
 
@@ -155,7 +155,7 @@ export type SpaceProtocolSessionParams = {
 export enum AuthStatus {
   INITIAL = 'INITIAL',
   SUCCESS = 'SUCCESS',
-  FAILURE = 'FAILURE'
+  FAILURE = 'FAILURE',
 }
 
 // TODO(dmaretskyi): Move to a separate file.
@@ -210,8 +210,8 @@ export class SpaceProtocolSession implements WireProtocol {
         onAuthFailure: () => {
           log.warn('Auth failed');
           this._authStatus = AuthStatus.FAILURE;
-        }
-      })
+        },
+      }),
     );
     this._teleport.addExtension('dxos.mesh.teleport.replicator', this.replicator);
   }