@dxos/echo-pipeline 0.4.8-next.fff1521 → 0.4.9-main.4fd47cd

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.4.8-next.fff1521",
+  "version": "0.4.9-main.4fd47cd",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -36,38 +36,38 @@
   ],
   "dependencies": {
     "crc-32": "^1.2.2",
-    "@dxos/async": "0.4.8-next.fff1521",
-    "@dxos/automerge": "0.4.8-next.fff1521",
-    "@dxos/codec-protobuf": "0.4.8-next.fff1521",
-    "@dxos/context": "0.4.8-next.fff1521",
-    "@dxos/credentials": "0.4.8-next.fff1521",
-    "@dxos/crypto": "0.4.8-next.fff1521",
-    "@dxos/document-model": "0.4.8-next.fff1521",
-    "@dxos/debug": "0.4.8-next.fff1521",
-    "@dxos/echo-db": "0.4.8-next.fff1521",
-    "@dxos/feed-store": "0.4.8-next.fff1521",
-    "@dxos/hypercore": "0.4.8-next.fff1521",
-    "@dxos/invariant": "0.4.8-next.fff1521",
-    "@dxos/keyring": "0.4.8-next.fff1521",
-    "@dxos/keys": "0.4.8-next.fff1521",
-    "@dxos/log": "0.4.8-next.fff1521",
-    "@dxos/model-factory": "0.4.8-next.fff1521",
-    "@dxos/network-manager": "0.4.8-next.fff1521",
-    "@dxos/messaging": "0.4.8-next.fff1521",
-    "@dxos/node-std": "0.4.8-next.fff1521",
-    "@dxos/protocols": "0.4.8-next.fff1521",
-    "@dxos/random-access-storage": "0.4.8-next.fff1521",
-    "@dxos/rpc": "0.4.8-next.fff1521",
-    "@dxos/teleport": "0.4.8-next.fff1521",
-    "@dxos/teleport-extension-automerge-replicator": "0.4.8-next.fff1521",
-    "@dxos/teleport-extension-gossip": "0.4.8-next.fff1521",
-    "@dxos/teleport-extension-object-sync": "0.4.8-next.fff1521",
-    "@dxos/teleport-extension-replicator": "0.4.8-next.fff1521",
-    "@dxos/text-model": "0.4.8-next.fff1521",
-    "@dxos/timeframe": "0.4.8-next.fff1521",
-    "@dxos/tracing": "0.4.8-next.fff1521",
-    "@dxos/typings": "0.4.8-next.fff1521",
-    "@dxos/util": "0.4.8-next.fff1521"
+    "@dxos/async": "0.4.9-main.4fd47cd",
+    "@dxos/codec-protobuf": "0.4.9-main.4fd47cd",
+    "@dxos/automerge": "0.4.9-main.4fd47cd",
+    "@dxos/context": "0.4.9-main.4fd47cd",
+    "@dxos/credentials": "0.4.9-main.4fd47cd",
+    "@dxos/crypto": "0.4.9-main.4fd47cd",
+    "@dxos/document-model": "0.4.9-main.4fd47cd",
+    "@dxos/echo-db": "0.4.9-main.4fd47cd",
+    "@dxos/feed-store": "0.4.9-main.4fd47cd",
+    "@dxos/debug": "0.4.9-main.4fd47cd",
+    "@dxos/hypercore": "0.4.9-main.4fd47cd",
+    "@dxos/keyring": "0.4.9-main.4fd47cd",
+    "@dxos/keys": "0.4.9-main.4fd47cd",
+    "@dxos/invariant": "0.4.9-main.4fd47cd",
+    "@dxos/log": "0.4.9-main.4fd47cd",
+    "@dxos/messaging": "0.4.9-main.4fd47cd",
+    "@dxos/model-factory": "0.4.9-main.4fd47cd",
+    "@dxos/network-manager": "0.4.9-main.4fd47cd",
+    "@dxos/node-std": "0.4.9-main.4fd47cd",
+    "@dxos/rpc": "0.4.9-main.4fd47cd",
+    "@dxos/protocols": "0.4.9-main.4fd47cd",
+    "@dxos/random-access-storage": "0.4.9-main.4fd47cd",
+    "@dxos/teleport-extension-automerge-replicator": "0.4.9-main.4fd47cd",
+    "@dxos/teleport": "0.4.9-main.4fd47cd",
+    "@dxos/teleport-extension-gossip": "0.4.9-main.4fd47cd",
+    "@dxos/teleport-extension-object-sync": "0.4.9-main.4fd47cd",
+    "@dxos/text-model": "0.4.9-main.4fd47cd",
+    "@dxos/teleport-extension-replicator": "0.4.9-main.4fd47cd",
+    "@dxos/timeframe": "0.4.9-main.4fd47cd",
+    "@dxos/tracing": "0.4.9-main.4fd47cd",
+    "@dxos/typings": "0.4.9-main.4fd47cd",
+    "@dxos/util": "0.4.9-main.4fd47cd"
   },
   "devDependencies": {
     "fast-check": "^3.15.1",

package/src/space/control-pipeline.ts

@@ -78,7 +78,9 @@ export class ControlPipeline {
         queueMicrotask(async () => {
           try {
             const feed = await feedProvider(info.key);
-            await this._pipeline.addFeed(feed);
+            if (!this._pipeline.hasFeed(feed.key)) {
+              await this._pipeline.addFeed(feed);
+            }
           } catch (err: any) {
             log.catch(err);
           }

package/src/space/data-pipeline.ts

@@ -10,7 +10,7 @@ import {
   type SpecificCredential,
   checkCredentialType,
 } from '@dxos/credentials';
-import { getStateMachineFromItem, ItemManager, TYPE_PROPERTIES } from '@dxos/echo-db';
+import { ItemManager } from '@dxos/echo-db';
 import { type FeedWriter } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
@@ -19,8 +19,6 @@ import { type ModelFactory } from '@dxos/model-factory';
 import { CancelledError, type DataPipelineProcessed } from '@dxos/protocols';
 import { type CreateEpochRequest } from '@dxos/protocols/proto/dxos/client/services';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
-import { type ObjectSnapshot } from '@dxos/protocols/proto/dxos/echo/model/document';
 import { type SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { type Credential, type Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
@@ -50,16 +48,6 @@ export type DataPipelineParams = {
   onPipelineCreated: (pipeline: Pipeline) => Promise<void>;
 };
 
-/**
- * Number of mutations since the last snapshot before we automatically create another snapshot.
- */
-const MESSAGES_PER_SNAPSHOT = 10;
-
-/**
- * Minimum time between automatic snapshots.
- */
-const AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5_000;
-
 /**
  * Minimum time in MS between recording latest timeframe in metadata.
  */
@@ -194,7 +182,6 @@ export class DataPipeline implements CredentialProcessor {
 
     // NOTE: Make sure the processing is stopped BEFORE we save the snapshot.
     try {
-      await this._saveCache();
       if (this._pipeline) {
         await this._saveTargetTimeframe(this._pipeline.state.timeframe);
       }
@@ -293,28 +280,6 @@ export class DataPipeline implements CredentialProcessor {
     this._targetTimeframe = newTimeframe;
   }
 
-  private async _saveCache() {
-    const cache: SpaceCache = {};
-
-    try {
-      // Add properties to cache.
-      const propertiesItem = this.itemManager.items.find(
-        (item) =>
-          item.modelMeta?.type === 'dxos.org/model/document' &&
-          // TODO(burdon): Document?
-          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === TYPE_PROPERTIES,
-      );
-      if (propertiesItem) {
-        cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot() as ObjectSnapshot;
-      }
-    } catch (err) {
-      log.warn('Failed to cache properties', err);
-    }
-
-    // Save cache.
-    await this._params.metadataStore.setCache(this._params.spaceKey, cache);
-  }
-
   private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
     if (!this._pipeline?.state.reachedTarget) {
       return;
@@ -327,13 +292,6 @@ export class DataPipeline implements CredentialProcessor {
 
       await this._saveTargetTimeframe(timeframe);
     }
-
-    if (
-      Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL &&
-      timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT
-    ) {
-      await this._saveCache();
-    }
   }
 
   private async _processEpochInSeparateTask(epoch: SpecificCredential<Epoch>) {
@@ -419,7 +377,6 @@ export class DataPipeline implements CredentialProcessor {
 
   private async _flush() {
     try {
-      await this._saveCache();
       if (this._pipeline) {
         await this._saveTargetTimeframe(this._pipeline.state.timeframe);
       }

package/src/testing/index.ts

@@ -6,4 +6,3 @@ export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
 export * from './util';
-export * from './database-test-rig';

package/src/testing/util.ts

@@ -4,16 +4,14 @@
 
 import { asyncTimeout } from '@dxos/async';
 import { DocumentModel } from '@dxos/document-model';
-import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
+import { ItemManager } from '@dxos/echo-db';
 import { MockFeedWriter } from '@dxos/feed-store/testing';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
-import { AutomergeHost } from '../automerge';
-import { DatabaseHost, type DataServiceHost, DataServiceImpl, DataServiceSubscriptions } from '../db-host';
+import { DatabaseHost } from '../db-host';
 import { type DataPipeline } from '../space';
 
 export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
@@ -41,28 +39,6 @@ export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
   };
 };
 
-export const createRemoteDatabaseFromDataServiceHost = async (
-  modelFactory: ModelFactory,
-  dataServiceHost: DataServiceHost,
-) => {
-  const dataServiceSubscriptions = new DataServiceSubscriptions();
-  const automergeHost = new AutomergeHost({
-    directory: createStorage({ type: StorageType.RAM }).createDirectory(),
-  });
-  const dataService = new DataServiceImpl(dataServiceSubscriptions, automergeHost);
-
-  const spaceKey = PublicKey.random();
-  await dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
-
-  const itemManager = new ItemManager(modelFactory);
-  const backend = new DatabaseProxy({ service: dataService, itemManager, spaceKey });
-  await backend.open(new ModelFactory().registerModel(DocumentModel));
-  return {
-    itemManager,
-    backend,
-  };
-};
-
 export const testLocalDatabase = async (create: DataPipeline, check: DataPipeline = create) => {
   const objectId = PublicKey.random().toHex();
   await create.databaseHost!.getWriteStream()?.write({