@dxos/echo-pipeline 0.4.8-main.05fda5d → 0.4.8-main.06fa7e4

package/dist/types/src/testing/util.d.ts

@@ -1,14 +1,10 @@
-import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
+import { ItemManager } from '@dxos/echo-db';
 import { ModelFactory } from '@dxos/model-factory';
-import { DatabaseHost, type DataServiceHost } from '../db-host';
+import { DatabaseHost } from '../db-host';
 import { type DataPipeline } from '../space';
 export declare const createMemoryDatabase: (modelFactory: ModelFactory) => Promise<{
     backend: DatabaseHost;
     itemManager: ItemManager;
 }>;
-export declare const createRemoteDatabaseFromDataServiceHost: (modelFactory: ModelFactory, dataServiceHost: DataServiceHost) => Promise<{
-    itemManager: ItemManager;
-    backend: DatabaseProxy;
-}>;
 export declare const testLocalDatabase: (create: DataPipeline, check?: DataPipeline) => Promise<void>;
 //# sourceMappingURL=util.d.ts.map

package/dist/types/src/testing/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/testing/util.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG3D,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAMnD,OAAO,EAAE,YAAY,EAAE,KAAK,eAAe,EAA6C,MAAM,YAAY,CAAC;AAC3G,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,eAAO,MAAM,oBAAoB,iBAAwB,YAAY;;;EAuBpE,CAAC;AAEF,eAAO,MAAM,uCAAuC,iBACpC,YAAY,mBACT,eAAe;;;EAkBjC,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAAkB,YAAY,UAAS,YAAY,kBAmBhF,CAAC"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/testing/util.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAG5C,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,KAAK,YAAY,EAAE,MAAM,UAAU,CAAC;AAE7C,eAAO,MAAM,oBAAoB,iBAAwB,YAAY;;;EAuBpE,CAAC;AAEF,eAAO,MAAM,iBAAiB,WAAkB,YAAY,UAAS,YAAY,kBAmBhF,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.4.8-main.05fda5d",
+  "version": "0.4.8-main.06fa7e4",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -36,38 +36,38 @@
   ],
   "dependencies": {
     "crc-32": "^1.2.2",
-    "@dxos/codec-protobuf": "0.4.8-main.05fda5d",
-    "@dxos/async": "0.4.8-main.05fda5d",
-    "@dxos/context": "0.4.8-main.05fda5d",
-    "@dxos/crypto": "0.4.8-main.05fda5d",
-    "@dxos/credentials": "0.4.8-main.05fda5d",
-    "@dxos/automerge": "0.4.8-main.05fda5d",
-    "@dxos/document-model": "0.4.8-main.05fda5d",
-    "@dxos/debug": "0.4.8-main.05fda5d",
-    "@dxos/feed-store": "0.4.8-main.05fda5d",
-    "@dxos/echo-db": "0.4.8-main.05fda5d",
-    "@dxos/hypercore": "0.4.8-main.05fda5d",
-    "@dxos/invariant": "0.4.8-main.05fda5d",
-    "@dxos/keys": "0.4.8-main.05fda5d",
-    "@dxos/log": "0.4.8-main.05fda5d",
-    "@dxos/keyring": "0.4.8-main.05fda5d",
-    "@dxos/messaging": "0.4.8-main.05fda5d",
-    "@dxos/network-manager": "0.4.8-main.05fda5d",
-    "@dxos/model-factory": "0.4.8-main.05fda5d",
-    "@dxos/node-std": "0.4.8-main.05fda5d",
-    "@dxos/random-access-storage": "0.4.8-main.05fda5d",
-    "@dxos/protocols": "0.4.8-main.05fda5d",
-    "@dxos/teleport": "0.4.8-main.05fda5d",
-    "@dxos/rpc": "0.4.8-main.05fda5d",
-    "@dxos/teleport-extension-automerge-replicator": "0.4.8-main.05fda5d",
-    "@dxos/teleport-extension-gossip": "0.4.8-main.05fda5d",
-    "@dxos/teleport-extension-replicator": "0.4.8-main.05fda5d",
-    "@dxos/timeframe": "0.4.8-main.05fda5d",
-    "@dxos/tracing": "0.4.8-main.05fda5d",
-    "@dxos/text-model": "0.4.8-main.05fda5d",
-    "@dxos/typings": "0.4.8-main.05fda5d",
-    "@dxos/util": "0.4.8-main.05fda5d",
-    "@dxos/teleport-extension-object-sync": "0.4.8-main.05fda5d"
+    "@dxos/async": "0.4.8-main.06fa7e4",
+    "@dxos/automerge": "0.4.8-main.06fa7e4",
+    "@dxos/codec-protobuf": "0.4.8-main.06fa7e4",
+    "@dxos/context": "0.4.8-main.06fa7e4",
+    "@dxos/credentials": "0.4.8-main.06fa7e4",
+    "@dxos/crypto": "0.4.8-main.06fa7e4",
+    "@dxos/echo-db": "0.4.8-main.06fa7e4",
+    "@dxos/document-model": "0.4.8-main.06fa7e4",
+    "@dxos/debug": "0.4.8-main.06fa7e4",
+    "@dxos/feed-store": "0.4.8-main.06fa7e4",
+    "@dxos/invariant": "0.4.8-main.06fa7e4",
+    "@dxos/hypercore": "0.4.8-main.06fa7e4",
+    "@dxos/keyring": "0.4.8-main.06fa7e4",
+    "@dxos/keys": "0.4.8-main.06fa7e4",
+    "@dxos/log": "0.4.8-main.06fa7e4",
+    "@dxos/messaging": "0.4.8-main.06fa7e4",
+    "@dxos/model-factory": "0.4.8-main.06fa7e4",
+    "@dxos/node-std": "0.4.8-main.06fa7e4",
+    "@dxos/network-manager": "0.4.8-main.06fa7e4",
+    "@dxos/protocols": "0.4.8-main.06fa7e4",
+    "@dxos/random-access-storage": "0.4.8-main.06fa7e4",
+    "@dxos/rpc": "0.4.8-main.06fa7e4",
+    "@dxos/teleport-extension-automerge-replicator": "0.4.8-main.06fa7e4",
+    "@dxos/teleport-extension-object-sync": "0.4.8-main.06fa7e4",
+    "@dxos/teleport": "0.4.8-main.06fa7e4",
+    "@dxos/teleport-extension-gossip": "0.4.8-main.06fa7e4",
+    "@dxos/teleport-extension-replicator": "0.4.8-main.06fa7e4",
+    "@dxos/text-model": "0.4.8-main.06fa7e4",
+    "@dxos/tracing": "0.4.8-main.06fa7e4",
+    "@dxos/timeframe": "0.4.8-main.06fa7e4",
+    "@dxos/typings": "0.4.8-main.06fa7e4",
+    "@dxos/util": "0.4.8-main.06fa7e4"
   },
   "devDependencies": {
     "fast-check": "^3.15.1",

package/src/automerge/automerge-host.ts

@@ -13,7 +13,7 @@ import {
 } from '@dxos/automerge/automerge-repo';
 import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
 import { type Stream } from '@dxos/codec-protobuf';
-import { Context, cancelWithContext } from '@dxos/context';
+import { Context } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { idCodec } from '@dxos/protocols';
@@ -31,7 +31,7 @@ import { MeshNetworkAdapter } from './mesh-network-adapter';
 export type { DocumentId };
 
 export interface MetadataMethods {
-  markDirty(id: string, lastAvailableHash: string): Promise<void>;
+  markDirty(idToLastHash: Map<string, string>): Promise<void>;
 }
 
 export type AutomergeHostParams = {
@@ -98,15 +98,13 @@ export class AutomergeHost {
         }
 
         try {
-          // experimental_spaceKey is set on old documents, new ones are created with doc.access.spaceKey
-          const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
-          if (!rawSpaceKey) {
+          const spaceKey = getSpaceKeyFromDoc(doc);
+          if (!spaceKey) {
             log('space key not found for share policy check', { peerId, documentId });
             return false;
           }
 
-          const spaceKey = PublicKey.from(rawSpaceKey);
-          const authorizedDevices = this._authorizedDevices.get(spaceKey);
+          const authorizedDevices = this._authorizedDevices.get(PublicKey.from(spaceKey));
 
           // TODO(mykola): Hack, stop abusing `peerMetadata` field.
           const deviceKeyHex = (this.repo.peerMetadataByPeerId[peerId] as any)?.dxos_deviceKey;
@@ -164,8 +162,7 @@ export class AutomergeHost {
   }
 
   private _onUpdate(event: DocHandleChangePayload<any>) {
-    const spaceKey = event.doc.access?.spaceKey;
-    if (!spaceKey) {
+    if (this._metadata == null) {
       return;
     }
 
@@ -180,24 +177,16 @@ export class AutomergeHost {
       return;
     }
 
-    const markingDirtyPromise = Promise.all(
-      objectIds.map(async (objectId) => {
-        await cancelWithContext(
-          this._ctx,
-          this._metadata!.markDirty(
-            idCodec.encode({ documentId: event.handle.documentId, objectId }),
-            lastAvailableHash,
-          ),
-        );
-      }),
-    )
+    const encodedIds = objectIds.map((objectId) => idCodec.encode({ documentId: event.handle.documentId, objectId }));
+    const idToLastHash = new Map(encodedIds.map((id) => [id, lastAvailableHash]));
+    const markingDirtyPromise = this._metadata
+      .markDirty(idToLastHash)
       .then(() => {
         this._updatingMetadata.delete(event.handle.documentId);
       })
       .catch((err: Error) => {
-        !this._ctx.disposed && log.catch(err);
+        this._ctx.disposed && log.catch(err);
       });
-
     this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
   }
 
@@ -285,3 +274,13 @@ const getInlineChanges = (event: DocHandleChangePayload<any>) => {
   }
   return [...inlineChangedObjectIds];
 };
+
+export const getSpaceKeyFromDoc = (doc: any): string | null => {
+  // experimental_spaceKey is set on old documents, new ones are created with doc.access.spaceKey
+  const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;
+  if (rawSpaceKey == null) {
+    return null;
+  }
+
+  return String(rawSpaceKey);
+};

package/src/automerge/automerge-storage-adapter.ts

@@ -5,20 +5,18 @@
 // Copyright 2023 DXOS.org
 //
 
-import { StorageAdapter, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
+import { type Chunk, type StorageKey, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
 import { type Directory } from '@dxos/random-access-storage';
 import { arrayToBuffer, bufferToArray } from '@dxos/util';
 
-export class AutomergeStorageAdapter extends StorageAdapter {
+export class AutomergeStorageAdapter implements StorageAdapterInterface {
   // TODO(mykola): Hack for restricting automerge Repo to access storage if Host is `closed`.
   //               Automerge Repo do not have any lifetime management.
   private _state: 'opened' | 'closed' = 'opened';
 
-  constructor(private readonly _directory: Directory) {
-    super();
-  }
+  constructor(private readonly _directory: Directory) {}
 
-  override async load(key: StorageKey): Promise<Uint8Array | undefined> {
+  async load(key: StorageKey): Promise<Uint8Array | undefined> {
     if (this._state !== 'opened') {
       return undefined;
     }
@@ -32,7 +30,7 @@ export class AutomergeStorageAdapter extends StorageAdapter {
     return bufferToArray(buffer);
   }
 
-  override async save(key: StorageKey, data: Uint8Array): Promise<void> {
+  async save(key: StorageKey, data: Uint8Array): Promise<void> {
     if (this._state !== 'opened') {
       return undefined;
     }
@@ -44,7 +42,7 @@ export class AutomergeStorageAdapter extends StorageAdapter {
     await file.flush?.();
   }
 
-  override async remove(key: StorageKey): Promise<void> {
+  async remove(key: StorageKey): Promise<void> {
     if (this._state !== 'opened') {
       return undefined;
     }
@@ -54,7 +52,7 @@ export class AutomergeStorageAdapter extends StorageAdapter {
     await file.destroy();
   }
 
-  override async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
     if (this._state !== 'opened') {
       return [];
     }
@@ -75,7 +73,7 @@ export class AutomergeStorageAdapter extends StorageAdapter {
     );
   }
 
-  override async removeRange(keyPrefix: StorageKey): Promise<void> {
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
     if (this._state !== 'opened') {
       return undefined;
     }

package/src/space/control-pipeline.ts

@@ -78,7 +78,9 @@ export class ControlPipeline {
         queueMicrotask(async () => {
           try {
             const feed = await feedProvider(info.key);
-            await this._pipeline.addFeed(feed);
+            if (!this._pipeline.hasFeed(feed.key)) {
+              await this._pipeline.addFeed(feed);
+            }
           } catch (err: any) {
             log.catch(err);
           }

package/src/space/data-pipeline.ts

@@ -10,7 +10,7 @@ import {
   type SpecificCredential,
   checkCredentialType,
 } from '@dxos/credentials';
-import { getStateMachineFromItem, ItemManager, TYPE_PROPERTIES } from '@dxos/echo-db';
+import { ItemManager } from '@dxos/echo-db';
 import { type FeedWriter } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { type PublicKey } from '@dxos/keys';
@@ -19,8 +19,6 @@ import { type ModelFactory } from '@dxos/model-factory';
 import { CancelledError, type DataPipelineProcessed } from '@dxos/protocols';
 import { type CreateEpochRequest } from '@dxos/protocols/proto/dxos/client/services';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { type SpaceCache } from '@dxos/protocols/proto/dxos/echo/metadata';
-import { type ObjectSnapshot } from '@dxos/protocols/proto/dxos/echo/model/document';
 import { type SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
 import { type Credential, type Epoch } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { Timeframe } from '@dxos/timeframe';
@@ -50,16 +48,6 @@ export type DataPipelineParams = {
   onPipelineCreated: (pipeline: Pipeline) => Promise<void>;
 };
 
-/**
- * Number of mutations since the last snapshot before we automatically create another snapshot.
- */
-const MESSAGES_PER_SNAPSHOT = 10;
-
-/**
- * Minimum time between automatic snapshots.
- */
-const AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL = 5_000;
-
 /**
  * Minimum time in MS between recording latest timeframe in metadata.
  */
@@ -194,7 +182,6 @@ export class DataPipeline implements CredentialProcessor {
 
     // NOTE: Make sure the processing is stopped BEFORE we save the snapshot.
     try {
-      await this._saveCache();
       if (this._pipeline) {
         await this._saveTargetTimeframe(this._pipeline.state.timeframe);
       }
@@ -293,28 +280,6 @@ export class DataPipeline implements CredentialProcessor {
     this._targetTimeframe = newTimeframe;
   }
 
-  private async _saveCache() {
-    const cache: SpaceCache = {};
-
-    try {
-      // Add properties to cache.
-      const propertiesItem = this.itemManager.items.find(
-        (item) =>
-          item.modelMeta?.type === 'dxos.org/model/document' &&
-          // TODO(burdon): Document?
-          (getStateMachineFromItem(item)?.snapshot() as ObjectSnapshot).type === TYPE_PROPERTIES,
-      );
-      if (propertiesItem) {
-        cache.properties = getStateMachineFromItem(propertiesItem)?.snapshot() as ObjectSnapshot;
-      }
-    } catch (err) {
-      log.warn('Failed to cache properties', err);
-    }
-
-    // Save cache.
-    await this._params.metadataStore.setCache(this._params.spaceKey, cache);
-  }
-
   private async _noteTargetStateIfNeeded(timeframe: Timeframe) {
     if (!this._pipeline?.state.reachedTarget) {
       return;
@@ -327,13 +292,6 @@ export class DataPipeline implements CredentialProcessor {
 
       await this._saveTargetTimeframe(timeframe);
     }
-
-    if (
-      Date.now() - this._lastSnapshotSaveTime > AUTOMATIC_SNAPSHOT_DEBOUNCE_INTERVAL &&
-      timeframe.totalMessages() - this._lastAutomaticSnapshotTimeframe.totalMessages() > MESSAGES_PER_SNAPSHOT
-    ) {
-      await this._saveCache();
-    }
   }
 
   private async _processEpochInSeparateTask(epoch: SpecificCredential<Epoch>) {
@@ -419,7 +377,6 @@ export class DataPipeline implements CredentialProcessor {
 
   private async _flush() {
     try {
-      await this._saveCache();
       if (this._pipeline) {
         await this._saveTargetTimeframe(this._pipeline.state.timeframe);
       }

package/src/testing/index.ts

@@ -6,4 +6,3 @@ export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
 export * from './util';
-export * from './database-test-rig';

package/src/testing/util.ts

@@ -4,16 +4,14 @@
 
 import { asyncTimeout } from '@dxos/async';
 import { DocumentModel } from '@dxos/document-model';
-import { DatabaseProxy, ItemManager } from '@dxos/echo-db';
+import { ItemManager } from '@dxos/echo-db';
 import { MockFeedWriter } from '@dxos/feed-store/testing';
 import { PublicKey } from '@dxos/keys';
 import { ModelFactory } from '@dxos/model-factory';
 import { type DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
-import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { Timeframe } from '@dxos/timeframe';
 
-import { AutomergeHost } from '../automerge';
-import { DatabaseHost, type DataServiceHost, DataServiceImpl, DataServiceSubscriptions } from '../db-host';
+import { DatabaseHost } from '../db-host';
 import { type DataPipeline } from '../space';
 
 export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
@@ -41,28 +39,6 @@ export const createMemoryDatabase = async (modelFactory: ModelFactory) => {
   };
 };
 
-export const createRemoteDatabaseFromDataServiceHost = async (
-  modelFactory: ModelFactory,
-  dataServiceHost: DataServiceHost,
-) => {
-  const dataServiceSubscriptions = new DataServiceSubscriptions();
-  const automergeHost = new AutomergeHost({
-    directory: createStorage({ type: StorageType.RAM }).createDirectory(),
-  });
-  const dataService = new DataServiceImpl(dataServiceSubscriptions, automergeHost);
-
-  const spaceKey = PublicKey.random();
-  await dataServiceSubscriptions.registerSpace(spaceKey, dataServiceHost);
-
-  const itemManager = new ItemManager(modelFactory);
-  const backend = new DatabaseProxy({ service: dataService, itemManager, spaceKey });
-  await backend.open(new ModelFactory().registerModel(DocumentModel));
-  return {
-    itemManager,
-    backend,
-  };
-};
-
 export const testLocalDatabase = async (create: DataPipeline, check: DataPipeline = create) => {
   const objectId = PublicKey.random().toHex();
   await create.databaseHost!.getWriteStream()?.write({