@dxos/echo-pipeline 0.1.36-next.ef27157 → 0.1.36

package/src/space/data-pipeline.ts

@@ -4,7 +4,7 @@
 
 import assert from 'node:assert';
 
-import { Event, scheduleTask, synchronized, trackLeaks } from '@dxos/async';
+import { scheduleTask, synchronized, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { FeedInfo } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
@@ -17,7 +17,7 @@ import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { Timeframe } from '@dxos/timeframe';
 
 import { createMappedFeedWriter } from '../common';
-import { DatabaseBackendHost, SnapshotManager } from '../dbhost';
+import { DatabaseHost, SnapshotManager } from '../dbhost';
 import { MetadataStore } from '../metadata';
 import { Pipeline } from '../pipeline';
 
@@ -46,7 +46,7 @@ const MESSAGES_PER_SNAPSHOT = 10;
 const AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5000;
 
 /**
- * Minimum time between recording latest timeframe in metadata.
+ * Minimum time in MS between recording latest timeframe in metadata.
  */
 const TIMEFRAME_SAVE_DEBOUNCE_INTERVAL = 500;
 
@@ -71,12 +71,13 @@ export class DataPipeline {
   private _lastAutomaticSnapshotTimeframe = new Timeframe();
   private _isOpen = false;
 
-  public readonly onTimeframeReached = new Event<Timeframe>();
+  private _lastTimeframeSaveTime = 0;
+  private _lastSnapshotSaveTime = 0;
 
   constructor(private readonly _params: DataPipelineParams) {}
 
   public _itemManager!: ItemManager;
-  public databaseBackend?: DatabaseBackendHost;
+  public databaseBackend?: DatabaseHost;
 
   get isOpen() {
     return this._isOpen;
@@ -122,7 +123,7 @@ export class DataPipeline {
       this._pipeline.writer ?? failUndefined()
     );
 
-    this.databaseBackend = new DatabaseBackendHost(feedWriter, this._snapshot?.database);
+    this.databaseBackend = new DatabaseHost(feedWriter, this._snapshot?.database);
     this._itemManager = new ItemManager(this._params.modelFactory);
 
     // Connect pipeline to the database.
@@ -133,53 +134,9 @@ export class DataPipeline {
       await this._consumePipeline();
     });
 
-    this._createPeriodicSnapshots();
-
     this._isOpen = true;
   }
 
-  private _createPeriodicSnapshots() {
-    // Record last timeframe.
-    this.onTimeframeReached.debounce(TIMEFRAME_SAVE_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
-      await this._saveLatestTimeframe();
-    });
-
-    if (!DISABLE_SNAPSHOT_CACHE) {
-      this.onTimeframeReached.debounce(AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL).on(this._ctx, async () => {
-        const latestTimeframe = this._pipeline?.state.timeframe;
-        if (!latestTimeframe) {
-          return;
-        }
-
-        // Save snapshot.
-        if (
-          latestTimeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() >
-          MESSAGES_PER_SNAPSHOT
-        ) {
-          const snapshot = await this._saveSnapshot();
-          this._lastAutomaticSnapshotTimeframe = snapshot.timeframe ?? failUndefined();
-          log('save', { snapshot });
-        }
-      });
-    }
-  }
-
-  private async _saveSnapshot() {
-    const snapshot = await this.createSnapshot();
-    const snapshotKey = await this._params.snapshotManager.store(snapshot);
-    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
-    return snapshot;
-  }
-
-  private async _saveLatestTimeframe() {
-    const latestTimeframe = this._pipeline?.state.timeframe;
-    log('save latest timeframe', { latestTimeframe, spaceKey: this._params.spaceKey });
-    if (latestTimeframe) {
-      const newTimeframe = Timeframe.merge(this._targetTimeframe ?? new Timeframe(), latestTimeframe);
-      await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
-    }
-  }
-
   @synchronized
   async close() {
     if (!this._isOpen) {
@@ -188,28 +145,26 @@ export class DataPipeline {
     log('close');
     this._isOpen = false;
 
+    await this._ctx.dispose();
+    await this._pipeline?.stop();
+
+    // NOTE: Make sure the processing is stopped BEFORE we save the snapshot.
     try {
-      await this._saveLatestTimeframe();
-      await this._saveSnapshot();
+      if (this._pipeline) {
+        await this._saveTargetTimeframe(this._pipeline.state.timeframe);
+        if (!DISABLE_SNAPSHOT_CACHE) {
+          await this._saveSnapshot(this._pipeline.state.timeframe);
+        }
+      }
     } catch (err) {
       log.catch(err);
     }
-    await this._ctx.dispose();
-    await this._pipeline?.stop();
+
     await this.databaseBackend?.close();
     await this._itemManager?.destroy();
     await this._params.snapshotManager.close();
   }
 
-  createSnapshot(): SpaceSnapshot {
-    assert(this.databaseBackend, 'Database backend is not initialized.');
-    return {
-      spaceKey: this._params.spaceKey.asUint8Array(),
-      timeframe: this._pipeline?.state.timeframe ?? new Timeframe(),
-      database: this.databaseBackend!.createSnapshot()
-    };
-  }
-
   private async _consumePipeline() {
     assert(this._pipeline, 'Pipeline is not initialized.');
     for await (const msg of this._pipeline.consume()) {
@@ -233,7 +188,9 @@ export class DataPipeline {
               memberKey: feedInfo.assertion.identityKey
             }
           });
-          this.onTimeframeReached.emit(data.timeframe);
+
+          // Timeframe clock is not updated yet
+          await this._noteTargetStateIfNeeded(this._pipeline.state.pendingTimeframe);
         }
       } catch (err: any) {
         log.catch(err);
@@ -241,6 +198,48 @@ export class DataPipeline {
     }
   }
 
+  private _createSnapshot(timeframe: Timeframe): SpaceSnapshot {
+    assert(this.databaseBackend, 'Database backend is not initialized.');
+    return {
+      spaceKey: this._params.spaceKey.asUint8Array(),
+      timeframe,
+      database: this.databaseBackend!.createSnapshot()
+    };
+  }
+
+  private async _saveSnapshot(timeframe: Timeframe) {
+    const snapshot = await this._createSnapshot(timeframe);
+    const snapshotKey = await this._params.snapshotManager.store(snapshot);
+    await this._params.metadataStore.setSpaceSnapshot(this._params.spaceKey, snapshotKey);
+    return snapshot;
+  }
+
+  private async _saveTargetTimeframe(timeframe: Timeframe) {
+    const newTimeframe = Timeframe.merge(this._targetTimeframe ?? new Timeframe(), timeframe);
+    await this._params.metadataStore.setSpaceLatestTimeframe(this._params.spaceKey, newTimeframe);
+    this._targetTimeframe = newTimeframe;
+  }
+
+  private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
+    if (Date.now() - this._lastTimeframeSaveTime > TIMEFRAME_SAVE_DEBOUNCE_INTERVAL) {
+      this._lastTimeframeSaveTime = Date.now();
+
+      await this._saveTargetTimeframe(timeframe);
+    }
+
+    if (
+      !DISABLE_SNAPSHOT_CACHE &&
+      Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL &&
+      timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT
+    ) {
+      this._lastSnapshotSaveTime = Date.now();
+
+      const snapshot = await this._saveSnapshot(timeframe);
+      this._lastAutomaticSnapshotTimeframe = snapshot.timeframe ?? failUndefined();
+      log('save', { snapshot });
+    }
+  }
+
   async waitUntilTimeframe(timeframe: Timeframe) {
     assert(this._pipeline, 'Pipeline is not initialized.');
     await this._pipeline.state.waitUntilTimeframe(timeframe);

package/src/space/space-manager.ts

@@ -48,6 +48,7 @@ export class SpaceManager {
   private readonly _feedStore: FeedStore<FeedMessage>;
   private readonly _networkManager: NetworkManager;
   private readonly _instanceId = PublicKey.random().toHex();
+  public _traceParent?: string;
 
   constructor({ feedStore, networkManager }: SpaceManagerParams) {
     // TODO(burdon): Assert.
@@ -62,7 +63,7 @@ export class SpaceManager {
 
   @synchronized
   async open() {
-    log.trace('dxos.echo.space-manager', trace.begin({ id: this._instanceId }));
+    log.trace('dxos.echo.space-manager', trace.begin({ id: this._instanceId, parentId: this._traceParent }));
   }
 
   @synchronized

package/src/testing/database-test-rig.ts

@@ -4,7 +4,7 @@
 
 import { Event } from '@dxos/async';
 import { DocumentModel } from '@dxos/document-model';
-import { DatabaseBackendProxy, ItemManager } from '@dxos/echo-db';
+import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
 import { FeedMessageBlock } from '@dxos/protocols';
@@ -14,7 +14,7 @@ import { TextModel } from '@dxos/text-model';
 import { Timeframe } from '@dxos/timeframe';
 import { ComplexMap, isNotNullOrUndefined } from '@dxos/util';
 
-import { DatabaseBackendHost } from '../dbhost';
+import { DatabaseHost } from '../dbhost';
 
 const SPACE_KEY = PublicKey.random();
 
@@ -33,9 +33,9 @@ export class DatabaseTestPeer {
   public readonly modelFactory = new ModelFactory().registerModel(DocumentModel).registerModel(TextModel);
 
   public items!: ItemManager;
-  public proxy!: DatabaseBackendProxy;
+  public proxy!: DatabaseProxy;
 
-  public host!: DatabaseBackendHost;
+  public host!: DatabaseHost;
   public hostItems!: ItemManager;
 
   //
@@ -64,7 +64,7 @@ export class DatabaseTestPeer {
 
   async open() {
     this.hostItems = new ItemManager(this.modelFactory);
-    this.host = new DatabaseBackendHost(
+    this.host = new DatabaseHost(
       {
         write: async (message) => {
           const seq =
@@ -86,7 +86,7 @@ export class DatabaseTestPeer {
     );
     await this.host.open(this.hostItems, this.modelFactory);
 
-    this.proxy = new DatabaseBackendProxy(this.host.createDataServiceHost(), SPACE_KEY);
+    this.proxy = new DatabaseProxy(this.host.createDataServiceHost(), SPACE_KEY);
     this.items = new ItemManager(this.modelFactory);
     await this.proxy.open(this.items, this.modelFactory);
   }

package/src/testing/util.ts

@@ -4,19 +4,19 @@
 
 import { asyncTimeout } from '@dxos/async';
 import { DocumentModel } from '@dxos/document-model';
-import { DatabaseBackendProxy, ItemManager } from '@dxos/echo-db';
+import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
 import { MockFeedWriter } from '@dxos/feed-store/testing';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
 import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { Timeframe } from '@dxos/timeframe';
 
-import { DatabaseBackendHost, DataServiceHost, DataServiceImpl, DataServiceSubscriptions } from '../dbhost';
+import { DatabaseHost, DataServiceHost, DataServiceImpl, DataServiceSubscriptions } from '../dbhost';
 import { DataPipeline } from '../space';
 
 export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
   const feed = new MockFeedWriter<DataMessage>();
-  const backend = new DatabaseBackendHost(feed, undefined);
+  const backend = new DatabaseHost(feed, undefined);
 
   feed.written.on(([data, meta]) =>
     backend.echoProcessor({
@@ -47,7 +47,7 @@ export const createRemoteDatabaseFromDataServiceHost = async (
   const spaceKey = PublicKey.random();
   dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
 
-  const backend = new DatabaseBackendProxy(dataService, spaceKey);
+  const backend = new DatabaseProxy(dataService, spaceKey);
   const itemManager = new ItemManager(modelFactory);
   await backend.open(itemManager, new ModelFactory().registerModel(DocumentModel));
   return {

package/src/tests/database-unit.test.ts

@@ -104,4 +104,17 @@ describe('database (unit)', () => {
     // TODO(dmaretskyi): Helper functions to compare state.
     expect(peer1.items.getItem(id)!.state).toEqual(peer1.items.getItem(id)!.state);
   });
+
+  // TODO(dmaretskyi): Flush is broken in this test database fixture.
+  test
+    .skip('flush', async () => {
+      const builder = new DatabaseTestBuilder();
+      const peer = await builder.createPeer();
+
+      const id = PublicKey.random().toHex();
+      peer.proxy.mutate(genesisMutation(id, DocumentModel.meta.type));
+      peer.confirm();
+      await peer.proxy.flush();
+    })
+    .timeout(100);
 });

package/src/tests/database.test.ts

@@ -5,11 +5,15 @@
 import expect from 'expect';
 
 import { DocumentModel, MutationBuilder } from '@dxos/document-model';
-import { createModelMutation, encodeModelMutation, genesisMutation } from '@dxos/echo-db';
+import { createModelMutation, DatabaseProxy, encodeModelMutation, genesisMutation, ItemManager } from '@dxos/echo-db';
+import { TestBuilder as FeedTestBuilder } from '@dxos/feed-store/testing';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
+import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
 import { test } from '@dxos/test';
 
+import { createMappedFeedWriter } from '../common';
+import { DatabaseHost, DataServiceImpl, DataServiceSubscriptions } from '../dbhost';
 import { createMemoryDatabase, createRemoteDatabaseFromDataServiceHost } from '../testing';
 
 const createDatabase = async () => {
@@ -23,6 +27,29 @@ const createDatabase = async () => {
   return proxy;
 };
 
+const createDatabaseWithFeeds = async () => {
+  const modelFactory = new ModelFactory().registerModel(DocumentModel);
+
+  const feedTestBuilder = new FeedTestBuilder();
+  const feedStore = feedTestBuilder.createFeedStore();
+  const feed = await feedStore.openFeed(await feedTestBuilder.keyring.createKey(), { writable: true });
+
+  const writer = createMappedFeedWriter((data: DataMessage) => ({ data }), feed.createFeedWriter());
+  const host = new DatabaseHost(writer);
+  await host.open(new ItemManager(modelFactory), new ModelFactory().registerModel(DocumentModel));
+
+  const dataServiceSubscriptions = new DataServiceSubscriptions();
+  const dataService = new DataServiceImpl(dataServiceSubscriptions);
+
+  const spaceKey = PublicKey.random();
+  dataServiceSubscriptions.registerSpace(spaceKey, host.createDataServiceHost());
+
+  const proxy = new DatabaseProxy(dataService, spaceKey);
+  await proxy.open(new ItemManager(modelFactory), new ModelFactory().registerModel(DocumentModel));
+
+  return { proxy, host };
+};
+
 describe('database', () => {
   describe('proxy-service mode', () => {
     test('create object', async () => {
@@ -49,5 +76,16 @@ describe('database', () => {
       await result.batch.waitToBeProcessed();
       expect(database.itemManager.getItem(id)!.state.data.test).toEqual(42);
     });
+
+    // TODO(dmaretskyi): Flush is broken in this mock database.
+    test
+      .skip('flush', async () => {
+        const { proxy } = await createDatabaseWithFeeds();
+
+        const id = PublicKey.random().toHex();
+        proxy.mutate(genesisMutation(id, DocumentModel.meta.type));
+        await proxy.flush();
+      })
+      .timeout(100);
   });
 });