@dxos/echo-pipeline 0.5.9-main.1c1903d → 0.5.9-main.21b00b3

package/src/automerge/automerge-storage-adapter.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2024 DXOS.org
+//
+//
+// Copyright 2023 DXOS.org
+//
+
+import { type Chunk, type StorageKey, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
+import { type Directory } from '@dxos/random-access-storage';
+import { arrayToBuffer, bufferToArray } from '@dxos/util';
+
+export class AutomergeStorageAdapter implements StorageAdapterInterface {
+  // TODO(mykola): Hack for restricting automerge Repo to access storage if Host is `closed`.
+  //               Automerge Repo do not have any lifetime management.
+  private _state: 'opened' | 'closed' = 'opened';
+
+  constructor(private readonly _directory: Directory) {}
+
+  async load(key: StorageKey): Promise<Uint8Array | undefined> {
+    if (this._state !== 'opened') {
+      return undefined;
+    }
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    const { size } = await file.stat();
+    if (!size || size === 0) {
+      return undefined;
+    }
+    const buffer = await file.read(0, size);
+    return bufferToArray(buffer);
+  }
+
+  async save(key: StorageKey, data: Uint8Array): Promise<void> {
+    if (this._state !== 'opened') {
+      return undefined;
+    }
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    await file.write(0, arrayToBuffer(data));
+    await file.truncate?.(data.length);
+
+    await file.flush?.();
+  }
+
+  async remove(key: StorageKey): Promise<void> {
+    if (this._state !== 'opened') {
+      return undefined;
+    }
+    // TODO(dmaretskyi): Better deletion.
+    const filename = this._getFilename(key);
+    const file = this._directory.getOrCreateFile(filename);
+    await file.destroy();
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    if (this._state !== 'opened') {
+      return [];
+    }
+    const filename = this._getFilename(keyPrefix);
+    const entries = await this._directory.list();
+    return Promise.all(
+      entries
+        .filter((entry) => entry.startsWith(filename))
+        .map(async (entry): Promise<Chunk> => {
+          const file = this._directory.getOrCreateFile(entry);
+          const { size } = await file.stat();
+          const buffer = await file.read(0, size);
+          return {
+            key: this._getKeyFromFilename(entry),
+            data: bufferToArray(buffer),
+          };
+        }),
+    );
+  }
+
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
+    if (this._state !== 'opened') {
+      return undefined;
+    }
+    const filename = this._getFilename(keyPrefix);
+    const entries = await this._directory.list();
+    await Promise.all(
+      entries
+        .filter((entry) => entry.startsWith(filename))
+        .map(async (entry): Promise<void> => {
+          const file = this._directory.getOrCreateFile(entry);
+          await file.destroy();
+        }),
+    );
+  }
+
+  async close(): Promise<void> {
+    this._state = 'closed';
+  }
+
+  private _getFilename(key: StorageKey): string {
+    return key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-');
+  }
+
+  private _getKeyFromFilename(filename: string): StorageKey {
+    return filename.split('-').map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%'));
+  }
+}

package/src/automerge/automerge-storage/342/200/223wrapper.ts

@@ -0,0 +1,59 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type StorageKey, type Chunk, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
+import { type MaybePromise } from '@dxos/util';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+
+export type StorageCallbacks = {
+  beforeSave?: (path: string[]) => MaybePromise<void>;
+  afterSave?: (path: string[]) => MaybePromise<void>;
+};
+
+export type AutomergeStorageWrapperParams = {
+  storage: StorageAdapterInterface;
+  callbacks: StorageCallbacks;
+};
+
+/**
+ * Wrapper for automerge storage that adds callback on save.
+ */
+export class AutomergeStorageWrapper implements StorageAdapterInterface {
+  private readonly _storage: StorageAdapterInterface;
+  private readonly _callbacks: StorageCallbacks;
+
+  constructor({ storage, callbacks }: AutomergeStorageWrapperParams) {
+    this._storage = storage;
+    this._callbacks = callbacks;
+  }
+
+  async load(key: StorageKey): Promise<Uint8Array | undefined> {
+    return this._storage.load(key);
+  }
+
+  async save(key: StorageKey, value: Uint8Array): Promise<void> {
+    await this._callbacks.beforeSave?.(key);
+    await this._storage.save(key, value);
+    await this._callbacks.afterSave?.(key);
+  }
+
+  async remove(key: StorageKey): Promise<void> {
+    return this._storage.remove(key);
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    return this._storage.loadRange(keyPrefix);
+  }
+
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
+    return this._storage.removeRange(keyPrefix);
+  }
+
+  async close() {
+    if (this._storage instanceof AutomergeStorageAdapter) {
+      return this._storage.close();
+    }
+  }
+}

package/src/automerge/echo-network-adapter.ts

@@ -93,7 +93,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
       peerId: this.peerId,
       onConnectionOpen: this._onConnectionOpen.bind(this),
       onConnectionClosed: this._onConnectionClosed.bind(this),
-      onConnectionAuthScopeChanged: this._onConnectionAuthScopeChanged.bind(this),
       getContainingSpaceForDocument: this._params.getContainingSpaceForDocument,
     });
   }
@@ -102,6 +101,7 @@ export class EchoNetworkAdapter extends NetworkAdapter {
   async removeReplicator(replicator: EchoReplicator) {
     invariant(this._lifecycleState === LifecycleState.OPEN);
     invariant(this._replicators.has(replicator));
+    ('');
     await replicator.disconnect();
     this._replicators.delete(replicator);
   }
@@ -142,19 +142,13 @@ export class EchoNetworkAdapter extends NetworkAdapter {
     });
 
     log('emit peer-candidate', { peerId: connection.peerId });
-    this._emitPeerCandidate(connection);
-  }
-
-  /**
-   * Trigger doc-synchronizer shared documents set recalculation. Happens on peer-candidate.
-   * TODO(y): replace with a proper API call when sharePolicy update becomes supported by automerge-repo
-   */
-  private _onConnectionAuthScopeChanged(connection: ReplicatorConnection) {
-    log('Connection auth scope changed', { peerId: connection.peerId });
-    const entry = this._connections.get(connection.peerId as PeerId);
-    invariant(entry);
-    this.emit('peer-disconnected', { peerId: connection.peerId as PeerId });
-    this._emitPeerCandidate(connection);
+    this.emit('peer-candidate', {
+      peerId: connection.peerId as PeerId,
+      peerMetadata: {
+        // TODO(dmaretskyi): Refactor this.
+        dxos_peerSource: 'EchoNetworkAdapter',
+      } as any,
+    });
   }
 
   private _onConnectionClosed(connection: ReplicatorConnection) {
@@ -170,16 +164,6 @@ export class EchoNetworkAdapter extends NetworkAdapter {
 
     this._connections.delete(connection.peerId as PeerId);
   }
-
-  private _emitPeerCandidate(connection: ReplicatorConnection) {
-    this.emit('peer-candidate', {
-      peerId: connection.peerId as PeerId,
-      peerMetadata: {
-        // TODO(dmaretskyi): Refactor this.
-        dxos_peerSource: 'EchoNetworkAdapter',
-      } as any,
-    });
-  }
 }
 
 type ConnectionEntry = {

package/src/automerge/echo-replicator.ts

@@ -27,8 +27,6 @@ export interface EchoReplicatorContext {
 
   onConnectionClosed(connection: ReplicatorConnection): void;
 
-  onConnectionAuthScopeChanged(connection: ReplicatorConnection): void;
-
   getContainingSpaceForDocument(documentId: string): Promise<PublicKey | null>;
 }
 

package/src/automerge/index.ts

@@ -3,6 +3,7 @@
 //
 
 export * from './automerge-host';
+export * from './automerge-storage-adapter';
 export * from './automerge-doc-loader';
 export * from './leveldb-storage-adapter';
 export * from './local-host-network-adapter';

package/src/automerge/mesh-echo-replicator.ts

@@ -59,9 +59,7 @@ export class MeshEchoReplicator implements EchoReplicator {
         log('onRemoteConnected', { peerId: connection.peerId });
         invariant(this._context);
 
-        if (this._connectionsPerPeer.has(connection.peerId)) {
-          this._context.onConnectionAuthScopeChanged(connection);
-        } else {
+        if (!this._connectionsPerPeer.has(connection.peerId)) {
           this._connectionsPerPeer.set(connection.peerId, connection);
           await connection.enable();
           this._context.onConnectionOpen(connection);

package/src/automerge/migrations.ts

@@ -0,0 +1,42 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type StorageKey } from '@dxos/automerge/automerge-repo';
+import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
+import { type SublevelDB } from '@dxos/kv-store';
+import { log } from '@dxos/log';
+import { StorageType, type Directory } from '@dxos/random-access-storage';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+import { encodingOptions } from './leveldb-storage-adapter';
+
+export const levelMigration = async ({ db, directory }: { db: SublevelDB; directory: Directory }) => {
+  // Note: Make auto-migration from previous storage to leveldb here.
+  const isNewLevel = !(await db
+    .iterator<StorageKey, Uint8Array>({
+      ...encodingOptions,
+    })
+    .next());
+
+  if (!isNewLevel) {
+    return;
+  }
+
+  const oldStorageAdapter =
+    directory.type === StorageType.IDB
+      ? new IndexedDBStorageAdapter(directory.path, 'data')
+      : new AutomergeStorageAdapter(directory);
+
+  const chunks = await oldStorageAdapter.loadRange([]);
+  if (chunks.length === 0) {
+    return;
+  }
+
+  const batch = db.batch();
+  log.info('found chunks on old storage adapter', { chunks: chunks.length });
+  for (const { key, data } of await oldStorageAdapter.loadRange([])) {
+    data && batch.put<StorageKey, Uint8Array>(key, data, { ...encodingOptions });
+  }
+  await batch.write();
+};

package/src/automerge/storage-adapter.test.ts

@@ -4,117 +4,153 @@
 
 import { expect } from 'chai';
 
+import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
 import { randomBytes } from '@dxos/crypto';
 import { PublicKey } from '@dxos/keys';
 import { createTestLevel } from '@dxos/kv-store/testing';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, test } from '@dxos/test';
-import { arrayToBuffer, bufferToArray } from '@dxos/util';
+import { arrayToBuffer, bufferToArray, type MaybePromise } from '@dxos/util';
 
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
 
-describe('LevelDBStorageAdapter', () => {
-  const createAdapter = async (root?: string) => {
-    const level = createTestLevel(root);
-    await level.open();
-    const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
-    await adapter.open?.();
-
-    const close = async () => {
-      await adapter.close();
-      await level.close();
-    };
-    afterTest(close);
-    return {
-      adapter,
-      close,
-    };
-  };
-
-  const chunks = [
-    { key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
-    { key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
-    { key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
-    { key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
-  ];
-
-  test('should store and retrieve data', async () => {
-    const { adapter } = await createAdapter();
-
-    await adapter.save(chunks[0].key, chunks[0].data);
-    expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
-  });
-
-  test('loadRange return inputs with correct prefixes', async () => {
-    const { adapter } = await createAdapter();
-
-    for (const chunk of chunks) {
-      await adapter.save(chunk.key, chunk.data);
-    }
-
-    expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
-    expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
-    expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
-  });
-
-  test('deletion works', async () => {
-    const { adapter } = await createAdapter();
-
-    for (const chunk of chunks) {
-      await adapter.save(chunk.key, chunk.data);
-    }
-
-    await adapter.remove(['a', 'b', 'c', '1']);
-
-    expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
-    expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
-
-    await adapter.removeRange(['a', 'b', 'd']);
-
-    expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
-    expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
-    expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
-    expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
+const runTests = (
+  testNamespace: string,
+  /** Run per test. Expects automatic clean-up with `afterTest`. */
+  createAdapter: (root?: string) => MaybePromise<{
+    adapter: StorageAdapterInterface;
+    /** Would be called automatically with `afterTest`. Exposed for mid-test clean-up. */
+    close: () => MaybePromise<void>;
+  }>,
+) => {
+  describe(testNamespace, () => {
+    const chunks = [
+      { key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
+    ];
+
+    test('should store and retrieve data', async () => {
+      const { adapter } = await createAdapter();
+
+      await adapter.save(chunks[0].key, chunks[0].data);
+      expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
+    });
+
+    test('loadRange return inputs with correct prefixes', async () => {
+      const { adapter } = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
+      expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
+    });
+
+    test('deletion works', async () => {
+      const { adapter } = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      await adapter.remove(['a', 'b', 'c', '1']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
+
+      await adapter.removeRange(['a', 'b', 'd']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
+      expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
+      expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
+      expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
+    });
+
+    test('loadRange', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const { adapter, close } = await createAdapter(root);
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+        await close();
+      }
+
+      {
+        const { adapter } = await createAdapter(root);
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['one', 'two']);
+        expect(range.map((chunk) => chunk.key)).to.deep.eq([
+          ['test', '1'],
+          ['test', '2'],
+        ]);
+      }
+    });
+
+    test('removeRange', async () => {
+      const root = `/tmp/${randomBytes(16).toString('hex')}`;
+      {
+        const { adapter, close } = await createAdapter(root);
+        await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
+        await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
+        await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
+        await close();
+      }
+
+      {
+        const { adapter } = await createAdapter(root);
+        await adapter.removeRange(['test']);
+        const range = await adapter.loadRange(['test']);
+        expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq([]);
+        const range2 = await adapter.loadRange(['bar']);
+        expect(range2.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['bar']);
+        expect(range2.map((chunk) => chunk.key)).to.deep.eq([['bar', '1']]);
+      }
+    });
   });
+};
+
+/**
+ * Run tests for AutomergeStorageAdapter.
+ */
+runTests('AutomergeStorageAdapter', (root?: string) => {
+  const storage = createStorage({ type: root ? StorageType.NODE : StorageType.RAM, root });
+  const dir = storage.createDirectory('automerge');
+  const adapter = new AutomergeStorageAdapter(dir);
+
+  const close = async () => {
+    await adapter.close();
+    await storage.close();
+  };
+  afterTest(close);
 
-  test('loadRange', async () => {
-    const root = `/tmp/${randomBytes(16).toString('hex')}`;
-    {
-      const { adapter, close } = await createAdapter(root);
-      await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
-      await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
-      await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
-      await close();
-    }
-
-    {
-      const { adapter } = await createAdapter(root);
-      const range = await adapter.loadRange(['test']);
-      expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['one', 'two']);
-      expect(range.map((chunk) => chunk.key)).to.deep.eq([
-        ['test', '1'],
-        ['test', '2'],
-      ]);
-    }
-  });
+  return {
+    adapter,
+    close,
+  };
+});
 
-  test('removeRange', async () => {
-    const root = `/tmp/${randomBytes(16).toString('hex')}`;
-    {
-      const { adapter, close } = await createAdapter(root);
-      await adapter.save(['test', '1'], bufferToArray(Buffer.from('one')));
-      await adapter.save(['test', '2'], bufferToArray(Buffer.from('two')));
-      await adapter.save(['bar', '1'], bufferToArray(Buffer.from('bar')));
-      await close();
-    }
-
-    {
-      const { adapter } = await createAdapter(root);
-      await adapter.removeRange(['test']);
-      const range = await adapter.loadRange(['test']);
-      expect(range.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq([]);
-      const range2 = await adapter.loadRange(['bar']);
-      expect(range2.map((chunk) => arrayToBuffer(chunk.data!).toString())).to.deep.eq(['bar']);
-      expect(range2.map((chunk) => chunk.key)).to.deep.eq([['bar', '1']]);
-    }
-  });
+/**
+ * Run tests for LevelDBStorageAdapter.
+ */
+runTests('LevelDBStorageAdapter', async (root?: string) => {
+  const level = createTestLevel(root);
+  await level.open();
+  const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
+  await adapter.open?.();
+
+  const close = async () => {
+    await adapter.close();
+    await level.close();
+  };
+  afterTest(close);
+  return {
+    adapter,
+    close,
+  };
 });