@dxos/echo-pipeline 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

package/package.json

@@ -1,12 +1,16 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.8.4-main.ae835ea",
+  "version": "0.8.4-main.bc674ce",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "info@dxos.org",
-  "sideEffects": true,
+  "sideEffects": false,
   "type": "module",
   "exports": {
     ".": {
@@ -43,50 +47,56 @@
     "src"
   ],
   "dependencies": {
-    "@automerge/automerge": "3.1.2",
-    "@automerge/automerge-repo": "2.4.0",
+    "@automerge/automerge": "3.2.3",
+    "@automerge/automerge-repo": "2.5.1",
+    "@effect/sql": "0.48.6",
+    "@effect/sql-sqlite-node": "0.49.1",
     "crc-32": "^1.2.2",
-    "effect": "3.18.3",
+    "effect": "3.19.11",
     "level-transcoder": "^1.0.1",
     "lodash.isequal": "^4.5.0",
-    "@dxos/codec-protobuf": "0.8.4-main.ae835ea",
-    "@dxos/async": "0.8.4-main.ae835ea",
-    "@dxos/context": "0.8.4-main.ae835ea",
-    "@dxos/credentials": "0.8.4-main.ae835ea",
-    "@dxos/crypto": "0.8.4-main.ae835ea",
-    "@dxos/debug": "0.8.4-main.ae835ea",
-    "@dxos/echo-protocol": "0.8.4-main.ae835ea",
-    "@dxos/echo": "0.8.4-main.ae835ea",
-    "@dxos/edge-client": "0.8.4-main.ae835ea",
-    "@dxos/errors": "0.8.4-main.ae835ea",
-    "@dxos/feed-store": "0.8.4-main.ae835ea",
-    "@dxos/hypercore": "0.8.4-main.ae835ea",
-    "@dxos/invariant": "0.8.4-main.ae835ea",
-    "@dxos/keyring": "0.8.4-main.ae835ea",
-    "@dxos/keys": "0.8.4-main.ae835ea",
-    "@dxos/kv-store": "0.8.4-main.ae835ea",
-    "@dxos/indexing": "0.8.4-main.ae835ea",
-    "@dxos/log": "0.8.4-main.ae835ea",
-    "@dxos/messaging": "0.8.4-main.ae835ea",
-    "@dxos/network-manager": "0.8.4-main.ae835ea",
-    "@dxos/protocols": "0.8.4-main.ae835ea",
-    "@dxos/node-std": "0.8.4-main.ae835ea",
-    "@dxos/random-access-storage": "0.8.4-main.ae835ea",
-    "@dxos/teleport-extension-automerge-replicator": "0.8.4-main.ae835ea",
-    "@dxos/teleport-extension-gossip": "0.8.4-main.ae835ea",
-    "@dxos/teleport-extension-object-sync": "0.8.4-main.ae835ea",
-    "@dxos/timeframe": "0.8.4-main.ae835ea",
-    "@dxos/teleport": "0.8.4-main.ae835ea",
-    "@dxos/tracing": "0.8.4-main.ae835ea",
-    "@dxos/teleport-extension-replicator": "0.8.4-main.ae835ea",
-    "@dxos/typings": "0.8.4-main.ae835ea",
-    "@dxos/util": "0.8.4-main.ae835ea"
+    "@dxos/async": "0.8.4-main.bc674ce",
+    "@dxos/context": "0.8.4-main.bc674ce",
+    "@dxos/codec-protobuf": "0.8.4-main.bc674ce",
+    "@dxos/credentials": "0.8.4-main.bc674ce",
+    "@dxos/debug": "0.8.4-main.bc674ce",
+    "@dxos/crypto": "0.8.4-main.bc674ce",
+    "@dxos/echo": "0.8.4-main.bc674ce",
+    "@dxos/echo-protocol": "0.8.4-main.bc674ce",
+    "@dxos/edge-client": "0.8.4-main.bc674ce",
+    "@dxos/errors": "0.8.4-main.bc674ce",
+    "@dxos/effect": "0.8.4-main.bc674ce",
+    "@dxos/feed": "0.8.4-main.bc674ce",
+    "@dxos/feed-store": "0.8.4-main.bc674ce",
+    "@dxos/index-core": "0.8.4-main.bc674ce",
+    "@dxos/hypercore": "0.8.4-main.bc674ce",
+    "@dxos/invariant": "0.8.4-main.bc674ce",
+    "@dxos/indexing": "0.8.4-main.bc674ce",
+    "@dxos/keyring": "0.8.4-main.bc674ce",
+    "@dxos/keys": "0.8.4-main.bc674ce",
+    "@dxos/kv-store": "0.8.4-main.bc674ce",
+    "@dxos/log": "0.8.4-main.bc674ce",
+    "@dxos/messaging": "0.8.4-main.bc674ce",
+    "@dxos/network-manager": "0.8.4-main.bc674ce",
+    "@dxos/node-std": "0.8.4-main.bc674ce",
+    "@dxos/protocols": "0.8.4-main.bc674ce",
+    "@dxos/random-access-storage": "0.8.4-main.bc674ce",
+    "@dxos/sql-sqlite": "0.8.4-main.bc674ce",
+    "@dxos/teleport": "0.8.4-main.bc674ce",
+    "@dxos/teleport-extension-automerge-replicator": "0.8.4-main.bc674ce",
+    "@dxos/teleport-extension-object-sync": "0.8.4-main.bc674ce",
+    "@dxos/tracing": "0.8.4-main.bc674ce",
+    "@dxos/teleport-extension-gossip": "0.8.4-main.bc674ce",
+    "@dxos/teleport-extension-replicator": "0.8.4-main.bc674ce",
+    "@dxos/timeframe": "0.8.4-main.bc674ce",
+    "@dxos/util": "0.8.4-main.bc674ce",
+    "@dxos/typings": "0.8.4-main.bc674ce"
   },
   "devDependencies": {
     "@types/lodash.isequal": "^4.5.0",
     "fast-check": "^3.19.0",
     "get-port-please": "^3.1.1",
-    "@dxos/test-utils": "0.8.4-main.ae835ea"
+    "@dxos/test-utils": "0.8.4-main.bc674ce"
   },
   "publishConfig": {
     "access": "public"

package/src/automerge/automerge-host.test.ts

@@ -3,9 +3,11 @@
 //
 
 import { getHeads } from '@automerge/automerge';
-import type { DocumentId, Heads } from '@automerge/automerge-repo';
+import * as Automerge from '@automerge/automerge';
+import { type DocumentId, type Heads, generateAutomergeUrl, parseAutomergeUrl } from '@automerge/automerge-repo';
 import { describe, expect, onTestFinished, test } from 'vitest';
 
+import { Context } from '@dxos/context';
 import { IndexMetadataStore } from '@dxos/indexing';
 import { PublicKey } from '@dxos/keys';
 import type { LevelDB } from '@dxos/kv-store';
@@ -21,31 +23,51 @@ describe('AutomergeHost', () => {
   test('can create documents', async () => {
     const level = await createLevel();
     const host = await setupAutomergeHost({ level });
-    const handle = host.repo.create<any>();
+    const handle = await host.createDoc<any>();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
-    await host.repo.flush();
+    await host.flush();
     expect(handle.doc()!.text).toEqual('Hello world');
   });
 
   test('changes are preserved in storage', async () => {
     const level = await createLevel();
     const host = await setupAutomergeHost({ level });
-    const handle = host.repo.create();
+    const handle = await host.createDoc<any>();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
     const url = handle.url;
 
-    await host.repo.flush();
+    await host.flush();
     await host.close();
 
     const host2 = await setupAutomergeHost({ level });
-    const handle2 = await host2.repo.find<any>(url);
+    const handle2 = await host2.loadDoc<any>(Context.default(), url);
     await handle2.whenReady();
     expect(handle2.doc()!.text).toEqual('Hello world');
-    await host2.repo.flush();
+    await host2.flush();
+  });
+
+  test('load resolves when document is created from binary', async () => {
+    const level = await createLevel();
+    const host = await setupAutomergeHost({ level });
+
+    // Create a document to get its binary representation
+    const document = Automerge.from({ text: 'Hello world' });
+    const binary = Automerge.save(document);
+    const { documentId } = parseAutomergeUrl(generateAutomergeUrl());
+
+    // Start loading a non-existent document (should hang until created)
+    const loadPromise = host.loadDoc(Context.default(), documentId);
+
+    // Create the document from binary - this should resolve the load
+    const createdHandle = await host.createDoc(binary, { preserveHistory: true, documentId });
+
+    // The load should now resolve
+    const loadedHandle = await loadPromise;
+    expect(loadedHandle.doc()).toEqual(createdHandle.doc());
   });
 
   test('query single document heads', async () => {
@@ -53,7 +75,7 @@ describe('AutomergeHost', () => {
 
     const level = await createLevel(tmpPath);
     const host = await setupAutomergeHost({ level });
-    const handle = host.createDoc({ text: 'Hello world' });
+    const handle = await host.createDoc({ text: 'Hello world' });
     const expectedHeads = getHeads(handle.doc()!);
     await host.flush();
 
@@ -73,7 +95,7 @@ describe('AutomergeHost', () => {
 
     const level = await createLevel(tmpPath);
     const host = await setupAutomergeHost({ level });
-    const handles = range(2, () => host.createDoc({ text: 'Hello world' }));
+    const handles = await Promise.all(range(2, () => host.createDoc({ text: 'Hello world' })));
     const expectedHeads: (Heads | undefined)[] = handles.map((handle) => getHeads(handle.doc()!));
     await host.flush();
 
@@ -106,7 +128,7 @@ describe('AutomergeHost', () => {
     {
       const host2 = await setupAutomergeHost({ level: level2 });
       for (const i of range(NUM_DOCUMENTS)) {
-        const handle = host2.createDoc({ docIndex: i });
+        const handle = await host2.createDoc({ docIndex: i });
         documentIds.push(handle.documentId);
       }
       await host2.flush();
@@ -115,20 +137,21 @@ describe('AutomergeHost', () => {
 
     const host2 = await setupAutomergeHost({ level: level2 });
 
+    const network = await new TestReplicationNetwork().open();
+    await host1.addReplicator(await network.createReplicator());
+    await host2.addReplicator(await network.createReplicator());
+
     const collectionId = 'test-collection';
     await host1.updateLocalCollectionState(collectionId, documentIds);
     await host2.updateLocalCollectionState(collectionId, documentIds);
 
-    await using network = await new TestReplicationNetwork().open();
-    await host1.addReplicator(await network.createReplicator());
-    await host2.addReplicator(await network.createReplicator());
-
     for (const documentId of documentIds) {
       await expect.poll(() => host1.getHeads([documentId])).toEqual(await host2.getHeads([documentId]));
     }
 
     await host1.close();
     await host2.close();
+    await network.close();
   });
 
   const createLevel = async (tmpPath?: string) => {

package/src/automerge/automerge-host.ts

@@ -25,6 +25,7 @@ import {
   type StorageKey,
   interpretAsDocumentId,
 } from '@automerge/automerge-repo';
+import type * as Record from 'effect/Record';
 
 import { DeferredTask, Event, asyncTimeout } from '@dxos/async';
 import { Context, type Lifecycle, Resource, cancelWithContext } from '@dxos/context';
@@ -38,20 +39,20 @@ import { objectPointerCodec } from '@dxos/protocols';
 import { type SpaceSyncState } from '@dxos/protocols/proto/dxos/echo/service';
 import { type DocHeadsList, type FlushRequest } from '@dxos/protocols/proto/dxos/echo/service';
 import { trace } from '@dxos/tracing';
-import { ComplexSet, bufferToArray, isNonNullable, range } from '@dxos/util';
+import { ComplexSet, bufferToArray, defaultMap, isNonNullable, range } from '@dxos/util';
 
 import { type CollectionState, CollectionSynchronizer, diffCollectionState } from './collection-synchronizer';
 import { type EchoDataMonitor } from './echo-data-monitor';
 import { EchoNetworkAdapter, isEchoPeerMetadata } from './echo-network-adapter';
-import { type EchoReplicator, type RemoteDocumentExistenceCheckParams } from './echo-replicator';
+import { type EchoReplicator, type RemoteDocumentExistenceCheckProps } from './echo-replicator';
 import { HeadsStore } from './heads-store';
-import { type BeforeSaveParams, LevelDBStorageAdapter } from './leveldb-storage-adapter';
+import { type BeforeSaveProps, LevelDBStorageAdapter } from './leveldb-storage-adapter';
 
 export type PeerIdProvider = () => string | undefined;
 
 export type RootDocumentSpaceKeyProvider = (documentId: string) => PublicKey | undefined;
 
-export type AutomergeHostParams = {
+export type AutomergeHostProps = {
   db: LevelDB;
   indexMetadataStore: IndexMetadataStore;
   dataMonitor?: EchoDataMonitor;
@@ -65,6 +66,12 @@ export type AutomergeHostParams = {
 
 export type LoadDocOptions = {
   timeout?: number;
+
+  /**
+   * If true, the document will be fetched from the network if it is not found in the local storage.
+   * Setting this to false does not guarantee that the document will not be fetched from the network.
+   */
+  fetchFromNetwork?: boolean;
 };
 
 export type CreateDocOptions = {
@@ -72,6 +79,8 @@ export type CreateDocOptions = {
    * Import the document together with its history.
    */
   preserveHistory?: boolean;
+
+  documentId?: DocumentId;
 };
 
 export const FIND_PARAMS = {
@@ -91,6 +100,11 @@ const BUNDLE_SYNC_CONCURRENCY = 2;
  */
 const BUNDLE_SYNC_THRESHOLD = 50;
 
+/**
+ * Only announce documents that are known to require sync.
+ */
+const OPTIMIZED_SHARE_POLICY = true;
+
 /**
  * Abstracts over the AutomergeRepo.
  */
@@ -132,7 +146,29 @@ export class AutomergeHost extends Resource {
   public readonly documentsSaved = new Event();
 
   private readonly _headsUpdates = new Map<DocumentId, Heads>();
-  private _onHeadsChangedTask?: DeferredTask | undefined;
+  private _onHeadsChangedTask?: DeferredTask;
+
+  /**
+   * Documents created in this session.
+   */
+  private _createdDocuments = new Set<DocumentId>();
+
+  /**
+   * Documents that need to be synced based on the result of collection-sync.
+   */
+  private _documentsToSync = new Set<DocumentId>();
+
+  /**
+   * Documents that are not avaiale locally that should be requested.
+   */
+  private _documentsToRequest = new Set<DocumentId>();
+
+  /**
+   * Documents requested by remote peers.
+   */
+  private _documentsRequested = new Map<PeerId, Set<DocumentId>>();
+
+  private _sharePolicyChangedTask?: DeferredTask;
 
   constructor({
     db,
@@ -140,7 +176,7 @@ export class AutomergeHost extends Resource {
     dataMonitor,
     peerIdProvider,
     getSpaceKeyByRootDocumentId,
-  }: AutomergeHostParams) {
+  }: AutomergeHostProps) {
     super();
     this._db = db;
     this._storage = new LevelDBStorageAdapter({
@@ -158,6 +194,10 @@ export class AutomergeHost extends Resource {
       onCollectionStateReceived: this._onCollectionStateReceived.bind(this),
       monitor: dataMonitor,
     });
+    this._echoNetworkAdapter.documentRequested.on(({ peerId, documentId }) => {
+      defaultMap(this._documentsRequested, peerId, () => new Set()).add(documentId);
+      this._sharePolicyChangedTask!.schedule();
+    });
     this._headsStore = new HeadsStore({ db: db.sublevel('heads') });
     this._indexMetadataStore = indexMetadataStore;
     this._peerIdProvider = peerIdProvider;
@@ -178,7 +218,7 @@ export class AutomergeHost extends Resource {
     // Construct the automerge repo.
     this._repo = new Repo({
       peerId: this._peerId as PeerId,
-      sharePolicy: this._sharePolicy.bind(this),
+      shareConfig: this._shareConfig,
       storage: this._storage,
       network: [
         // Upstream swarm.
@@ -226,6 +266,11 @@ export class AutomergeHost extends Resource {
       );
     });
 
+    this._sharePolicyChangedTask = new DeferredTask(this._ctx, async () => {
+      log('share policy changed');
+      this._repo.shareConfigChanged();
+    });
+
     await this._echoNetworkAdapter.open();
     await this._collectionSynchronizer.open();
     await this._echoNetworkAdapter.open();
@@ -238,13 +283,7 @@ export class AutomergeHost extends Resource {
     await this._echoNetworkAdapter.close();
     this._syncTask = undefined;
     this._onHeadsChangedTask = undefined;
-  }
-
-  /**
-   * @deprecated To be abstracted away.
-   */
-  get repo(): Repo {
-    return this._repo;
+    this._sharePolicyChangedTask = undefined;
   }
 
   get peerId(): PeerId {
@@ -255,6 +294,10 @@ export class AutomergeHost extends Resource {
     return Object.keys(this._repo.handles).length;
   }
 
+  get handles(): Readonly<Record<DocumentId, DocHandle<any>>> {
+    return this._repo.handles;
+  }
+
   async addReplicator(replicator: EchoReplicator): Promise<void> {
     invariant(this.isOpen, 'AutomergeHost is not open');
     await this._echoNetworkAdapter.addReplicator(replicator);
@@ -276,7 +319,22 @@ export class AutomergeHost extends Resource {
       handle = this._repo.handles[documentId as DocumentId];
     }
     if (!handle) {
-      handle = await this._repo.find(documentId as DocumentId, FIND_PARAMS);
+      if (!opts?.fetchFromNetwork) {
+        handle = await this._repo.find(documentId as DocumentId, FIND_PARAMS);
+      } else {
+        try {
+          handle = await this._repo.find(documentId as DocumentId, {
+            allowableStates: ['ready', 'requesting', 'unavailable'],
+          });
+          if (handle.state === 'requesting' || handle.state === 'unavailable') {
+            this._documentsToRequest.add(handle.documentId);
+            this._sharePolicyChangedTask!.schedule();
+          }
+        } catch (err) {
+          log.error('failed to load document', { documentId, err });
+          throw err;
+        }
+      }
     }
 
     // `whenReady` creates a timeout so we guard it with an if to skip it if the handle is already ready.
@@ -302,11 +360,11 @@ export class AutomergeHost extends Resource {
   /**
    * Create new persisted document.
    */
-  createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): DocHandle<T> {
+  async createDoc<T>(initialValue?: T | Doc<T> | Uint8Array, opts?: CreateDocOptions): Promise<DocHandle<T>> {
     invariant(this.isOpen, 'AutomergeHost is not open');
     if (opts?.preserveHistory) {
       if (initialValue instanceof Uint8Array) {
-        return this._repo.import(initialValue);
+        return this._repo.import(initialValue, { docId: opts?.documentId });
       }
 
       if (!isAutomerge(initialValue)) {
@@ -314,13 +372,22 @@ export class AutomergeHost extends Resource {
       }
 
       // TODO(dmaretskyi): There's a more efficient way.
-      return this._repo.import(save(initialValue as Doc<T>));
+      const handle = this._repo.import(save(initialValue as Doc<T>), { docId: opts?.documentId });
+      this._createdDocuments.add(handle.documentId);
+      this._sharePolicyChangedTask!.schedule();
+      return handle as DocHandle<T>;
     } else {
       if (initialValue instanceof Uint8Array) {
         throw new Error('Cannot create document from Uint8Array without preserving history');
       }
 
-      return this._repo.create(initialValue);
+      if (opts?.documentId) {
+        throw new Error('Cannot prefil document id when not importing an existing doc');
+      }
+      const handle = await this._repo.create2<T>(initialValue);
+      this._createdDocuments.add(handle.documentId);
+      this._sharePolicyChangedTask!.schedule();
+      return handle;
     }
   }
 
@@ -373,24 +440,42 @@ export class AutomergeHost extends Resource {
     log('done re-indexing heads');
   }
 
-  // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
-  // Hosts, running in the worker, don't share documents unless requested by other peers.
-  // NOTE: If both peers return sharePolicy=false the replication will not happen
-  // https://github.com/automerge/automerge-repo/pull/292
-  private async _sharePolicy(peerId: PeerId, documentId?: DocumentId): Promise<boolean> {
-    if (!documentId) {
-      return false;
-    }
+  private readonly _shareConfig = {
+    access: async (peerId: PeerId, documentId: DocumentId): Promise<boolean> => {
+      return true;
+    },
 
-    const peerMetadata = this.repo.peerMetadataByPeerId[peerId];
-    if (isEchoPeerMetadata(peerMetadata)) {
-      return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
-    }
+    // TODO(dmaretskyi): Share based on HALO permissions and space affinity.
+    // Hosts, running in the worker, don't share documents unless requested by other peers.
+    // NOTE: If both peers return sharePolicy=false the replication will not happen
+    // https://github.com/automerge/automerge-repo/pull/292
+    // Called for all loaded documents so they could be advertised to the sync server.
+    announce: async (peerId: PeerId, documentId?: DocumentId): Promise<boolean> => {
+      if (!documentId) {
+        return false;
+      }
 
-    return false;
-  }
+      if (OPTIMIZED_SHARE_POLICY) {
+        if (
+          !this._createdDocuments.has(documentId) &&
+          !this._documentsToSync.has(documentId) &&
+          !this._documentsToRequest.has(documentId)
+        ) {
+          // Skip advertising documents that don't need to be synced.
+          return false;
+        }
+      }
+
+      const peerMetadata = this._repo.peerMetadataByPeerId[peerId];
+      if (isEchoPeerMetadata(peerMetadata)) {
+        return this._echoNetworkAdapter.shouldAdvertise(peerId, { documentId });
+      }
 
-  private async _beforeSave({ path, batch }: BeforeSaveParams): Promise<void> {
+      return false;
+    },
+  };
+
+  private async _beforeSave({ path, batch }: BeforeSaveProps): Promise<void> {
     const handle = this._repo.handles[path[0] as DocumentId];
     if (!handle || !handle.isReady()) {
       return;
@@ -452,7 +537,7 @@ export class AutomergeHost extends Resource {
     return this._repo.peers;
   }
 
-  private async _isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckParams): Promise<boolean> {
+  private async _isDocumentInRemoteCollection(params: RemoteDocumentExistenceCheckProps): Promise<boolean> {
     for (const collectionId of this._collectionSynchronizer.getRegisteredCollectionIds()) {
       const remoteCollections = this._collectionSynchronizer.getRemoteCollectionStates(collectionId);
       const remotePeerDocs = remoteCollections.get(params.peerId as PeerId)?.documents;
@@ -530,6 +615,13 @@ export class AutomergeHost extends Resource {
     return result;
   }
 
+  /**
+   * Iterate over all document heads stored on disk.
+   */
+  listDocumentHeads(): AsyncGenerator<{ documentId: DocumentId; heads: Heads }> {
+    return this._headsStore.iterateAll();
+  }
+
   //
   // Collection sync.
   //
@@ -684,12 +776,14 @@ export class AutomergeHost extends Resource {
 
     // Load the documents so they will start syncing.
     for (const documentId of toReplicateWithoutBatching) {
+      // Unless we track the document in "to sync" list, it will not be advertised.
+      this._documentsToSync!.add(documentId);
       this._repo.findWithProgress(documentId);
     }
+    this._sharePolicyChangedTask!.schedule();
   }
 
   // TODO(mykola): Add retries of batches https://gist.github.com/mykola-vrmchk/fde270259e9209fcbf1331e5abbf12cf
-  // TODO(mykola): Use effect to retry batches.
   private async _pushInBundles(
     peerId: PeerId,
     documentIds: DocumentId[],