@dxos/echo-pipeline 0.4.10-main.fd4f2a3 → 0.4.10-main.fd8ea31

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dxos/echo-pipeline",
-  "version": "0.4.10-main.fd4f2a3",
+  "version": "0.4.10-main.fd8ea31",
   "description": "ECHO database.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
@@ -35,36 +35,39 @@
     "src"
   ],
   "dependencies": {
+    "abstract-level": "^1.0.2",
     "crc-32": "^1.2.2",
-    "@dxos/async": "0.4.10-main.fd4f2a3",
-    "@dxos/codec-protobuf": "0.4.10-main.fd4f2a3",
-    "@dxos/context": "0.4.10-main.fd4f2a3",
-    "@dxos/credentials": "0.4.10-main.fd4f2a3",
-    "@dxos/debug": "0.4.10-main.fd4f2a3",
-    "@dxos/crypto": "0.4.10-main.fd4f2a3",
-    "@dxos/echo-db": "0.4.10-main.fd4f2a3",
-    "@dxos/feed-store": "0.4.10-main.fd4f2a3",
-    "@dxos/hypercore": "0.4.10-main.fd4f2a3",
-    "@dxos/invariant": "0.4.10-main.fd4f2a3",
-    "@dxos/keys": "0.4.10-main.fd4f2a3",
-    "@dxos/keyring": "0.4.10-main.fd4f2a3",
-    "@dxos/automerge": "0.4.10-main.fd4f2a3",
-    "@dxos/log": "0.4.10-main.fd4f2a3",
-    "@dxos/messaging": "0.4.10-main.fd4f2a3",
-    "@dxos/network-manager": "0.4.10-main.fd4f2a3",
-    "@dxos/node-std": "0.4.10-main.fd4f2a3",
-    "@dxos/protocols": "0.4.10-main.fd4f2a3",
-    "@dxos/random-access-storage": "0.4.10-main.fd4f2a3",
-    "@dxos/rpc": "0.4.10-main.fd4f2a3",
-    "@dxos/teleport": "0.4.10-main.fd4f2a3",
-    "@dxos/teleport-extension-automerge-replicator": "0.4.10-main.fd4f2a3",
-    "@dxos/teleport-extension-gossip": "0.4.10-main.fd4f2a3",
-    "@dxos/teleport-extension-object-sync": "0.4.10-main.fd4f2a3",
-    "@dxos/timeframe": "0.4.10-main.fd4f2a3",
-    "@dxos/teleport-extension-replicator": "0.4.10-main.fd4f2a3",
-    "@dxos/tracing": "0.4.10-main.fd4f2a3",
-    "@dxos/typings": "0.4.10-main.fd4f2a3",
-    "@dxos/util": "0.4.10-main.fd4f2a3"
+    "level": "^8.0.1",
+    "level-transcoder": "^1.0.1",
+    "@dxos/async": "0.4.10-main.fd8ea31",
+    "@dxos/automerge": "0.4.10-main.fd8ea31",
+    "@dxos/codec-protobuf": "0.4.10-main.fd8ea31",
+    "@dxos/context": "0.4.10-main.fd8ea31",
+    "@dxos/credentials": "0.4.10-main.fd8ea31",
+    "@dxos/debug": "0.4.10-main.fd8ea31",
+    "@dxos/crypto": "0.4.10-main.fd8ea31",
+    "@dxos/echo-schema": "0.4.10-main.fd8ea31",
+    "@dxos/feed-store": "0.4.10-main.fd8ea31",
+    "@dxos/keyring": "0.4.10-main.fd8ea31",
+    "@dxos/hypercore": "0.4.10-main.fd8ea31",
+    "@dxos/invariant": "0.4.10-main.fd8ea31",
+    "@dxos/keys": "0.4.10-main.fd8ea31",
+    "@dxos/log": "0.4.10-main.fd8ea31",
+    "@dxos/messaging": "0.4.10-main.fd8ea31",
+    "@dxos/network-manager": "0.4.10-main.fd8ea31",
+    "@dxos/protocols": "0.4.10-main.fd8ea31",
+    "@dxos/node-std": "0.4.10-main.fd8ea31",
+    "@dxos/random-access-storage": "0.4.10-main.fd8ea31",
+    "@dxos/rpc": "0.4.10-main.fd8ea31",
+    "@dxos/teleport-extension-automerge-replicator": "0.4.10-main.fd8ea31",
+    "@dxos/teleport": "0.4.10-main.fd8ea31",
+    "@dxos/teleport-extension-gossip": "0.4.10-main.fd8ea31",
+    "@dxos/teleport-extension-object-sync": "0.4.10-main.fd8ea31",
+    "@dxos/teleport-extension-replicator": "0.4.10-main.fd8ea31",
+    "@dxos/timeframe": "0.4.10-main.fd8ea31",
+    "@dxos/tracing": "0.4.10-main.fd8ea31",
+    "@dxos/util": "0.4.10-main.fd8ea31",
+    "@dxos/typings": "0.4.10-main.fd8ea31"
   },
   "devDependencies": {
     "fast-check": "^3.15.1",

package/src/automerge/automerge-host.test.ts

@@ -25,35 +25,50 @@ import { arrayToBuffer, bufferToArray } from '@dxos/util';
 import { AutomergeHost } from './automerge-host';
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { MeshNetworkAdapter } from './mesh-network-adapter';
+import { createTestLevel } from '../testing';
 
 describe('AutomergeHost', () => {
-  test('can create documents', () => {
-    const host = new AutomergeHost({ directory: createStorage({ type: StorageType.RAM }).createDirectory() });
+  test('can create documents', async () => {
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
+    const host = new AutomergeHost({
+      db: level.sublevel('automerge'),
+    });
+    await host.open();
+    afterTest(() => host.close());
 
     const handle = host.repo.create();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
+    await host.repo.flush();
     expect(handle.docSync().text).toEqual('Hello world');
   });
 
   test('changes are preserved in storage', async () => {
-    const storageDirectory = createStorage({ type: StorageType.RAM }).createDirectory();
+    const level = createTestLevel();
+    await level.open();
+    afterTest(() => level.close());
 
-    const host = new AutomergeHost({ directory: storageDirectory });
+    const host = new AutomergeHost({ db: level.sublevel('automerge') });
+    await host.open();
     const handle = host.repo.create();
     handle.change((doc: any) => {
       doc.text = 'Hello world';
     });
     const url = handle.url;
 
-    // TODO(dmaretskyi): Is there a way to know when automerge has finished saving?
-    await sleep(100);
+    await host.repo.flush();
+    await host.close();
 
-    const host2 = new AutomergeHost({ directory: storageDirectory });
+    const host2 = new AutomergeHost({ db: level.sublevel('automerge') });
+    await host2.open();
+    afterTest(() => host2.close());
     const handle2 = host2.repo.find(url);
     await handle2.whenReady();
     expect(handle2.docSync().text).toEqual('Hello world');
+    await host2.repo.flush();
   });
 
   test('basic networking', async () => {
@@ -168,7 +183,6 @@ describe('AutomergeHost', () => {
     const secondHandle = host.create();
     secondHandle.change((doc: any) => (doc.text = 'Hello world'));
     await host.find(secondHandle.url).whenReady();
-    // await sleep(100);
     allowedDocs.push(secondHandle.documentId);
 
     {

package/src/automerge/automerge-host.ts

@@ -2,83 +2,82 @@
 // Copyright 2023 DXOS.org
 //
 
-import { next as automerge, getHeads } from '@dxos/automerge/automerge';
-import {
-  Repo,
-  type PeerId,
-  type DocumentId,
-  type StorageKey,
-  type DocHandle,
-  type DocHandleChangePayload,
-} from '@dxos/automerge/automerge-repo';
-import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
+import { asyncTimeout } from '@dxos/async';
+import { next as automerge } from '@dxos/automerge/automerge';
+import { Repo, type DocumentId, type PeerId, type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
 import { type Stream } from '@dxos/codec-protobuf';
-import { Context } from '@dxos/context';
+import { Context, type Lifecycle } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { idCodec } from '@dxos/protocols';
 import {
   type FlushRequest,
   type HostInfo,
   type SyncRepoRequest,
   type SyncRepoResponse,
 } from '@dxos/protocols/proto/dxos/echo/service';
-import { StorageType, type Directory } from '@dxos/random-access-storage';
+import { type Directory } from '@dxos/random-access-storage';
 import { type AutomergeReplicator } from '@dxos/teleport-extension-automerge-replicator';
 import { trace } from '@dxos/tracing';
 import { ComplexMap, ComplexSet, defaultMap, mapValues } from '@dxos/util';
 
-import { AutomergeStorageAdapter } from './automerge-storage-adapter';
-import { AutomergeStorageWrapper } from './automerge-storage–wrapper';
+import { LevelDBStorageAdapter, type StorageCallbacks } from './leveldb-storage-adapter';
 import { LocalHostNetworkAdapter } from './local-host-network-adapter';
 import { MeshNetworkAdapter } from './mesh-network-adapter';
+import { levelMigration } from './migrations';
+import { type SubLevelDB } from './types';
 
+// TODO: Remove
 export type { DocumentId };
 
-export interface MetadataMethods {
-  markDirty(idToLastHash: Map<string, string>): Promise<void>;
-}
-
 export type AutomergeHostParams = {
-  directory: Directory;
-  metadata?: MetadataMethods;
+  db: SubLevelDB;
+  /**
+   * For migration purposes.
+   */
+  directory?: Directory;
+  storageCallbacks?: StorageCallbacks;
 };
 
 @trace.resource()
 export class AutomergeHost {
   private readonly _ctx = new Context();
-  private readonly _repo: Repo;
-  private readonly _meshNetwork: MeshNetworkAdapter;
-  private readonly _clientNetwork: LocalHostNetworkAdapter;
-  private readonly _storage: AutomergeStorageWrapper;
+  private readonly _directory?: Directory;
+  private readonly _db: SubLevelDB;
+  private readonly _storageCallbacks?: StorageCallbacks;
+
+  private _repo!: Repo;
+  private _meshNetwork!: MeshNetworkAdapter;
+  private _clientNetwork!: LocalHostNetworkAdapter;
+  private _storage!: StorageAdapterInterface & Lifecycle;
 
   @trace.info()
-  private readonly _peerId: string;
+  private _peerId!: string;
 
   /**
    * spaceKey -> deviceKey[]
    */
   private readonly _authorizedDevices = new ComplexMap<PublicKey, ComplexSet<PublicKey>>(PublicKey.hash);
 
-  private readonly _updatingMetadata = new Map<string, Promise<void>>();
-  private readonly _metadata?: MetadataMethods;
-
   public _requestedDocs = new Set<string>();
 
-  constructor({ directory, metadata }: AutomergeHostParams) {
-    this._metadata = metadata;
-    this._meshNetwork = new MeshNetworkAdapter();
-    this._clientNetwork = new LocalHostNetworkAdapter();
+  constructor({ directory, db, storageCallbacks }: AutomergeHostParams) {
+    this._directory = directory;
+    this._db = db;
+    this._storageCallbacks = storageCallbacks;
+  }
 
-    this._storage = new AutomergeStorageWrapper({
-      storage:
-        // TODO(mykola): Delete specific handling of IDB storage.
-        directory.type === StorageType.IDB
-          ? new IndexedDBStorageAdapter(directory.path, 'data')
-          : new AutomergeStorageAdapter(directory),
-      callbacks: { beforeSave: (params) => this._beforeSave(params) },
+  async open() {
+    // TODO(mykola): remove this before 0.6 release.
+    this._directory && (await levelMigration({ db: this._db, directory: this._directory }));
+    this._storage = new LevelDBStorageAdapter({
+      db: this._db,
+      callbacks: this._storageCallbacks,
     });
+    await this._storage.open?.();
     this._peerId = `host-${PublicKey.random().toHex()}` as PeerId;
+
+    this._meshNetwork = new MeshNetworkAdapter();
+    this._clientNetwork = new LocalHostNetworkAdapter();
     this._repo = new Repo({
       peerId: this._peerId as PeerId,
       network: [this._clientNetwork, this._meshNetwork],
@@ -138,59 +137,17 @@ export class AutomergeHost {
     this._clientNetwork.ready();
     this._meshNetwork.ready();
 
-    {
-      const listener = ({ handle }: { handle: DocHandle<any> }) => this._onDocument(handle);
-      this._repo.on('document', listener);
-      this._ctx.onDispose(() => {
-        this._repo.off('document', listener);
-        Object.values(this._repo.handles).forEach((handle) => handle.off('change'));
-      });
-    }
-  }
-
-  get repo(): Repo {
-    return this._repo;
+    await this._clientNetwork.whenConnected();
   }
 
-  private async _beforeSave(path: StorageKey) {
-    const id = path[0];
-    if (this._updatingMetadata.has(id)) {
-      return this._updatingMetadata.get(id);
-    }
-  }
-
-  private _onDocument(handle: DocHandle<any>) {
-    const listener = (event: DocHandleChangePayload<any>) => this._onUpdate(event);
-    handle.on('change', listener);
+  async close() {
+    await this._storage.close?.();
+    await this._clientNetwork.close();
+    await this._ctx.dispose();
   }
 
-  private _onUpdate(event: DocHandleChangePayload<any>) {
-    if (this._metadata == null) {
-      return;
-    }
-
-    const objectIds = getInlineChanges(event);
-    if (objectIds.length === 0) {
-      return;
-    }
-
-    const heads = getHeads(event.doc);
-    const lastAvailableHash = heads.join('');
-    if (!lastAvailableHash) {
-      return;
-    }
-
-    const encodedIds = objectIds.map((objectId) => idCodec.encode({ documentId: event.handle.documentId, objectId }));
-    const idToLastHash = new Map(encodedIds.map((id) => [id, lastAvailableHash]));
-    const markingDirtyPromise = this._metadata
-      .markDirty(idToLastHash)
-      .then(() => {
-        this._updatingMetadata.delete(event.handle.documentId);
-      })
-      .catch((err: Error) => {
-        this._ctx.disposed && log.catch(err);
-      });
-    this._updatingMetadata.set(event.handle.documentId, markingDirtyPromise);
+  get repo(): Repo {
+    return this._repo;
   }
 
   @trace.info({ depth: null })
@@ -224,12 +181,6 @@ export class AutomergeHost {
     return this._repo.peers;
   }
 
-  async close() {
-    await this._storage.close();
-    await this._clientNetwork.close();
-    await this._ctx.dispose();
-  }
-
   //
   // Methods for client-services.
   //
@@ -237,7 +188,13 @@ export class AutomergeHost {
   async flush({ documentIds }: FlushRequest): Promise<void> {
     // Note: Wait for all requested documents to be loaded/synced from thin-client.
     await Promise.all(documentIds?.map((id) => this._repo.find(id as DocumentId).whenReady()) ?? []);
-    await this._repo.flush(documentIds as DocumentId[]);
+
+    // TODO(dmaretskyi): Workaround until the flush issue gets resolved.
+    try {
+      await asyncTimeout(this._repo.flush(documentIds as DocumentId[]), 500);
+    } catch (err) {
+      log.warn('flush error', { documentIds, err });
+    }
   }
 
   syncRepo(request: SyncRepoRequest): Stream<SyncRepoResponse> {
@@ -266,24 +223,6 @@ export class AutomergeHost {
   }
 }
 
-// TODO(mykola): Reconcile with `getInlineAndLinkChanges` in AutomergeDB.
-const getInlineChanges = (event: DocHandleChangePayload<any>) => {
-  const inlineChangedObjectIds = new Set<string>();
-  for (const { path } of event.patches) {
-    if (path.length < 2) {
-      continue;
-    }
-    switch (path[0]) {
-      case 'objects':
-        if (path.length >= 2) {
-          inlineChangedObjectIds.add(path[1]);
-        }
-        break;
-    }
-  }
-  return [...inlineChangedObjectIds];
-};
-
 export const getSpaceKeyFromDoc = (doc: any): string | null => {
   // experimental_spaceKey is set on old documents, new ones are created with doc.access.spaceKey
   const rawSpaceKey = doc.access?.spaceKey ?? doc.experimental_spaceKey;

package/src/automerge/automerge-repo.test.ts

@@ -0,0 +1,29 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Repo } from '@dxos/automerge/automerge-repo';
+import { randomBytes } from '@dxos/crypto';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
+import { describe, test } from '@dxos/test';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+
+describe('AutomergeRepo', () => {
+  // Currently failing
+  test.skip('flush', async () => {
+    const repo = new Repo({
+      network: [],
+      storage: new AutomergeStorageAdapter(createStorage({ type: StorageType.NODE }).createDirectory()),
+    });
+    const handle = repo.create<{ field?: string }>();
+
+    for (let i = 0; i < 10; i++) {
+      const p = repo.flush([handle.documentId]);
+      handle.change((doc: any) => {
+        doc.field += randomBytes(1024).toString('hex');
+      });
+      await p;
+    }
+  });
+});

package/src/automerge/index.ts

@@ -5,6 +5,7 @@
 export * from './automerge-host';
 export * from './automerge-storage-adapter';
 export * from './automerge-doc-loader';
+export * from './leveldb-storage-adapter';
 export * from './local-host-network-adapter';
 export * from './mesh-network-adapter';
 export * from './types';

package/src/automerge/level.test.ts

@@ -0,0 +1,82 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+import { Level } from 'level';
+
+import { PublicKey } from '@dxos/keys';
+import { describe, openAndClose, test } from '@dxos/test';
+
+import { type SubLevelDB } from './types';
+import { createTestLevel } from '../testing';
+
+describe('Level', () => {
+  test('missing keys', async () => {
+    const level = createTestLevel();
+    await openAndClose(level);
+
+    expect(() => level.get('missing')).to.throw;
+  });
+
+  test('data persistance after reload', async () => {
+    const path = `/tmp/dxos-${PublicKey.random().toHex()}`;
+    const level = new Level<string, string>(path);
+    await level.open();
+
+    const key = 'name';
+    const value = 'Rich';
+    {
+      await level.put(key, value);
+      expect(await level.get(key)).to.equal(value);
+    }
+
+    await level.close();
+
+    {
+      const level = new Level<string, string>(path);
+      await level.open();
+      expect(await level.get(key)).to.equal(value);
+      await level.clear();
+      expect(() => level.get(key)).to.throw;
+    }
+  });
+
+  test('batch different sublevels', async () => {
+    const level = createTestLevel();
+    await openAndClose(level);
+
+    const first: SubLevelDB = level.sublevel('first');
+    const second: SubLevelDB = level.sublevel('second');
+
+    const batch = first.batch();
+
+    const key = 'key';
+    const value = 'first-level-value';
+    batch.put(key, value, { sublevel: second });
+    await batch.write();
+
+    expect(() => first.get(key)).to.throw;
+    expect(await level.sublevel('second').get(key)).to.equal(value);
+    expect(await second.get(key)).to.equal(value);
+    expect(await second.iterator().all()).to.deep.equal([[key, value]]);
+  });
+
+  test('sublevel prefixes', () => {
+    const level = createTestLevel();
+    expect(level.prefixKey('', 'utf8')).to.equal('');
+
+    {
+      const sublevel = level.sublevel('sublevel');
+      expect(sublevel.prefixKey('', 'utf8')).to.equal('!sublevel!');
+    }
+
+    {
+      const sublevel = level.sublevel('');
+      expect(sublevel.prefixKey('', 'utf8')).to.equal('!!');
+
+      const sublevel2 = sublevel.sublevel('');
+      expect(sublevel2.prefixKey('', 'utf8')).to.equal('!!!!');
+    }
+  });
+});

package/src/automerge/leveldb-storage-adapter.ts

@@ -0,0 +1,117 @@
+//
+// Copyright 2024 DXOS.org
+// s
+
+import { type MixedEncoding } from 'level-transcoder';
+
+import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
+import { LifecycleState, Resource } from '@dxos/context';
+import { type MaybePromise } from '@dxos/util';
+
+import { type BatchLevel, type SubLevelDB } from './types';
+
+export type LevelDBStorageAdapterParams = {
+  db: SubLevelDB;
+  callbacks?: StorageCallbacks;
+};
+
+export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
+
+export interface StorageCallbacks {
+  beforeSave(params: BeforeSaveParams): MaybePromise<void>;
+  afterSave(path: StorageKey): MaybePromise<void>;
+}
+
+export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
+  constructor(private readonly _params: LevelDBStorageAdapterParams) {
+    super();
+  }
+
+  async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
+    try {
+      if (this._lifecycleState !== LifecycleState.OPEN) {
+        // TODO(mykola): this should be an error.
+        return undefined;
+      }
+      return await this._params.db.get<StorageKey, Uint8Array>(keyArray, { ...encodingOptions });
+    } catch (err: any) {
+      if (isLevelDbNotFoundError(err)) {
+        return undefined;
+      }
+      throw err;
+    }
+  }
+
+  async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    const batch = this._params.db.batch();
+
+    await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
+    batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
+      ...encodingOptions,
+    });
+    await batch.write();
+
+    await this._params.callbacks?.afterSave?.(keyArray);
+  }
+
+  async remove(keyArray: StorageKey): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return [];
+    }
+    const result: Chunk[] = [];
+    for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
+      gte: keyPrefix,
+      lte: [...keyPrefix, '\uffff'],
+      ...encodingOptions,
+    })) {
+      result.push({
+        key,
+        data: value,
+      });
+    }
+    return result;
+  }
+
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    const batch = this._params.db.batch();
+
+    for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({
+      gte: keyPrefix,
+      lte: [...keyPrefix, '\uffff'],
+      ...encodingOptions,
+    })) {
+      batch.del<StorageKey>(key, { ...encodingOptions });
+    }
+    await batch.write();
+  }
+}
+
+const keyEncoder: MixedEncoding<StorageKey, Uint8Array, StorageKey> = {
+  encode: (key: StorageKey): Uint8Array =>
+    Buffer.from(key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-')),
+  decode: (key: Uint8Array): StorageKey =>
+    Buffer.from(key)
+      .toString()
+      .split('-')
+      .map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%')),
+};
+
+export const encodingOptions = {
+  keyEncoding: keyEncoder,
+  valueEncoding: 'buffer',
+};
+
+const isLevelDbNotFoundError = (err: any): boolean => err.code === 'LEVEL_NOT_FOUND';