@dxos/echo-pipeline 0.4.10-main.0302e13 → 0.4.10-main.05dbd64

package/src/automerge/level.test.ts

@@ -0,0 +1,82 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+import { Level } from 'level';
+
+import { PublicKey } from '@dxos/keys';
+import { describe, openAndClose, test } from '@dxos/test';
+
+import { type SubLevelDB } from './types';
+import { createTestLevel } from '../testing';
+
+describe('Level', () => {
+  test('missing keys', async () => {
+    const level = createTestLevel();
+    await openAndClose(level);
+
+    expect(() => level.get('missing')).to.throw;
+  });
+
+  test('data persistance after reload', async () => {
+    const path = `/tmp/dxos-${PublicKey.random().toHex()}`;
+    const level = new Level<string, string>(path);
+    await level.open();
+
+    const key = 'name';
+    const value = 'Rich';
+    {
+      await level.put(key, value);
+      expect(await level.get(key)).to.equal(value);
+    }
+
+    await level.close();
+
+    {
+      const level = new Level<string, string>(path);
+      await level.open();
+      expect(await level.get(key)).to.equal(value);
+      await level.clear();
+      expect(() => level.get(key)).to.throw;
+    }
+  });
+
+  test('batch different sublevels', async () => {
+    const level = createTestLevel();
+    await openAndClose(level);
+
+    const first: SubLevelDB = level.sublevel('first');
+    const second: SubLevelDB = level.sublevel('second');
+
+    const batch = first.batch();
+
+    const key = 'key';
+    const value = 'first-level-value';
+    batch.put(key, value, { sublevel: second });
+    await batch.write();
+
+    expect(() => first.get(key)).to.throw;
+    expect(await level.sublevel('second').get(key)).to.equal(value);
+    expect(await second.get(key)).to.equal(value);
+    expect(await second.iterator().all()).to.deep.equal([[key, value]]);
+  });
+
+  test('sublevel prefixes', () => {
+    const level = createTestLevel();
+    expect(level.prefixKey('', 'utf8')).to.equal('');
+
+    {
+      const sublevel = level.sublevel('sublevel');
+      expect(sublevel.prefixKey('', 'utf8')).to.equal('!sublevel!');
+    }
+
+    {
+      const sublevel = level.sublevel('');
+      expect(sublevel.prefixKey('', 'utf8')).to.equal('!!');
+
+      const sublevel2 = sublevel.sublevel('');
+      expect(sublevel2.prefixKey('', 'utf8')).to.equal('!!!!');
+    }
+  });
+});

package/src/automerge/leveldb-storage-adapter.ts

@@ -0,0 +1,117 @@
+//
+// Copyright 2024 DXOS.org
+// s
+
+import { type MixedEncoding } from 'level-transcoder';
+
+import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
+import { LifecycleState, Resource } from '@dxos/context';
+import { type MaybePromise } from '@dxos/util';
+
+import { type BatchLevel, type SubLevelDB } from './types';
+
+export type LevelDBStorageAdapterParams = {
+  db: SubLevelDB;
+  callbacks?: StorageCallbacks;
+};
+
+export type BeforeSaveParams = { path: StorageKey; batch: BatchLevel };
+
+export interface StorageCallbacks {
+  beforeSave(params: BeforeSaveParams): MaybePromise<void>;
+  afterSave(path: StorageKey): MaybePromise<void>;
+}
+
+export class LevelDBStorageAdapter extends Resource implements StorageAdapterInterface {
+  constructor(private readonly _params: LevelDBStorageAdapterParams) {
+    super();
+  }
+
+  async load(keyArray: StorageKey): Promise<Uint8Array | undefined> {
+    try {
+      if (this._lifecycleState !== LifecycleState.OPEN) {
+        // TODO(mykola): this should be an error.
+        return undefined;
+      }
+      return await this._params.db.get<StorageKey, Uint8Array>(keyArray, { ...encodingOptions });
+    } catch (err: any) {
+      if (isLevelDbNotFoundError(err)) {
+        return undefined;
+      }
+      throw err;
+    }
+  }
+
+  async save(keyArray: StorageKey, binary: Uint8Array): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    const batch = this._params.db.batch();
+
+    await this._params.callbacks?.beforeSave?.({ path: keyArray, batch });
+    batch.put<StorageKey, Uint8Array>(keyArray, Buffer.from(binary), {
+      ...encodingOptions,
+    });
+    await batch.write();
+
+    await this._params.callbacks?.afterSave?.(keyArray);
+  }
+
+  async remove(keyArray: StorageKey): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    await this._params.db.del<StorageKey>(keyArray, { ...encodingOptions });
+  }
+
+  async loadRange(keyPrefix: StorageKey): Promise<Chunk[]> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return [];
+    }
+    const result: Chunk[] = [];
+    for await (const [key, value] of this._params.db.iterator<StorageKey, Uint8Array>({
+      gte: keyPrefix,
+      lte: [...keyPrefix, '\uffff'],
+      ...encodingOptions,
+    })) {
+      result.push({
+        key,
+        data: value,
+      });
+    }
+    return result;
+  }
+
+  async removeRange(keyPrefix: StorageKey): Promise<void> {
+    if (this._lifecycleState !== LifecycleState.OPEN) {
+      return undefined;
+    }
+    const batch = this._params.db.batch();
+
+    for await (const [key] of this._params.db.iterator<StorageKey, Uint8Array>({
+      gte: keyPrefix,
+      lte: [...keyPrefix, '\uffff'],
+      ...encodingOptions,
+    })) {
+      batch.del<StorageKey>(key, { ...encodingOptions });
+    }
+    await batch.write();
+  }
+}
+
+const keyEncoder: MixedEncoding<StorageKey, Uint8Array, StorageKey> = {
+  encode: (key: StorageKey): Uint8Array =>
+    Buffer.from(key.map((k) => k.replaceAll('%', '%25').replaceAll('-', '%2D')).join('-')),
+  decode: (key: Uint8Array): StorageKey =>
+    Buffer.from(key)
+      .toString()
+      .split('-')
+      .map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%')),
+};
+
+export const encodingOptions = {
+  keyEncoding: keyEncoder,
+  valueEncoding: 'buffer',
+};
+
+const isLevelDbNotFoundError = (err: any): boolean => err.code === 'LEVEL_NOT_FOUND';

package/src/automerge/local-host-network-adapter.ts

@@ -6,7 +6,6 @@ import { Trigger } from '@dxos/async';
 import { NetworkAdapter, type Message, type PeerId, cbor } from '@dxos/automerge/automerge-repo';
 import { Stream } from '@dxos/codec-protobuf';
 import { invariant } from '@dxos/invariant';
-import { log } from '@dxos/log';
 import { type HostInfo, type SyncRepoRequest, type SyncRepoResponse } from '@dxos/protocols/proto/dxos/echo/service';
 
 type ClientSyncState = {
@@ -32,10 +31,17 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
     });
   }
 
-  private _connected = new Trigger();
+  private readonly _connected = new Trigger();
+  private _isConnected: boolean = false;
 
+  /**
+   * Called by `Repo` to connect to the network.
+   *
+   * @param peerId Our peer Id.
+   */
   override connect(peerId: PeerId): void {
     this.peerId = peerId;
+    this._isConnected = true;
     this._connected.wake();
     // No-op. Client always connects first
   }
@@ -56,6 +62,10 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
     // No-op
   }
 
+  async whenConnected(): Promise<void> {
+    await this._connected.wait({ timeout: 10_000 });
+  }
+
   syncRepo({ id, syncMessage }: SyncRepoRequest): Stream<SyncRepoResponse> {
     const peerId = this._getPeerId(id);
 
@@ -77,26 +87,22 @@ export class LocalHostNetworkAdapter extends NetworkAdapter {
         },
       });
 
-      this._connected
-        .wait({ timeout: 1_000 })
-        .then(() => {
-          this.emit('peer-candidate', {
-            peerMetadata: {},
-            peerId,
-          });
-        })
-        .catch((err) => log.catch(err));
+      invariant(this._isConnected);
+      this.emit('peer-candidate', {
+        peerMetadata: {},
+        peerId,
+      });
     });
   }
 
   async sendSyncMessage({ id, syncMessage }: SyncRepoRequest): Promise<void> {
-    await this._connected.wait({ timeout: 1_000 });
+    invariant(this._isConnected);
     const message = cbor.decode(syncMessage!) as Message;
     this.emit('message', message);
   }
 
   async getHostInfo(): Promise<HostInfo> {
-    await this._connected.wait({ timeout: 1_000 });
+    invariant(this._isConnected);
     invariant(this.peerId, 'Peer id not set.');
     return {
       peerId: this.peerId,

package/src/automerge/migrations.ts

@@ -0,0 +1,41 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type StorageKey } from '@dxos/automerge/automerge-repo';
+import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
+import { log } from '@dxos/log';
+import { StorageType, type Directory } from '@dxos/random-access-storage';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+import { encodingOptions } from './leveldb-storage-adapter';
+import { type SubLevelDB } from './types';
+
+export const levelMigration = async ({ db, directory }: { db: SubLevelDB; directory: Directory }) => {
+  // Note: Make automigration from previous storage to leveldb here.
+  const isNewLevel = !(await db
+    .iterator<StorageKey, Uint8Array>({
+      ...encodingOptions,
+    })
+    .next());
+
+  if (!isNewLevel) {
+    return;
+  }
+
+  const oldStorageAdapter =
+    directory.type === StorageType.IDB
+      ? new IndexedDBStorageAdapter(directory.path, 'data')
+      : new AutomergeStorageAdapter(directory);
+
+  const chunks = await oldStorageAdapter.loadRange([]);
+  if (chunks.length === 0) {
+    return;
+  }
+  const batch = db.batch();
+  log.info('found chunks on old storage adapter', { chunks: chunks.length });
+  for (const { key, data } of await oldStorageAdapter.loadRange([])) {
+    data && batch.put<StorageKey, Uint8Array>(key, data, { ...encodingOptions });
+  }
+  await batch.write();
+};

package/src/automerge/reference.ts

@@ -0,0 +1,31 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Reference } from '@dxos/echo-schema';
+
+export const REFERENCE_TYPE_TAG = 'dxos.echo.model.document.Reference';
+
+/**
+ * Reference as it is stored in Automerge document.
+ */
+export type EncodedReferenceObject = {
+  '@type': typeof REFERENCE_TYPE_TAG;
+  itemId: string | null;
+  protocol: string | null;
+  host: string | null;
+};
+
+export const encodeReference = (reference: Reference): EncodedReferenceObject => ({
+  '@type': REFERENCE_TYPE_TAG,
+  // NOTE: Automerge do not support undefined values, so we need to use null instead.
+  itemId: reference.itemId ?? null,
+  protocol: reference.protocol ?? null,
+  host: reference.host ?? null,
+});
+
+export const decodeReference = (value: any) =>
+  new Reference(value.itemId, value.protocol ?? undefined, value.host ?? undefined);
+
+export const isEncodedReferenceObject = (value: any): value is EncodedReferenceObject =>
+  typeof value === 'object' && value !== null && value['@type'] === REFERENCE_TYPE_TAG;

package/src/automerge/storage-adapter.test.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
+import { PublicKey } from '@dxos/keys';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
+import { afterTest, describe, openAndClose, test } from '@dxos/test';
+import { type MaybePromise } from '@dxos/util';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
+import { createTestLevel } from '../testing';
+
+const runTests = (
+  testNamespace: string,
+  /** Run per test. Expects automatic clean-up with `afterTest`. */ createAdapter: () => MaybePromise<StorageAdapterInterface>,
+) => {
+  describe(testNamespace, () => {
+    const chunks = [
+      { key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
+    ];
+
+    test('should store and retrieve data', async () => {
+      const adapter = await createAdapter();
+
+      await adapter.save(chunks[0].key, chunks[0].data);
+      expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
+    });
+
+    test('loadRange return inputs with correct prefixes', async () => {
+      const adapter = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
+      expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
+    });
+
+    test('deletion works', async () => {
+      const adapter = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      await adapter.remove(['a', 'b', 'c', '1']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
+
+      await adapter.removeRange(['a', 'b', 'd']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
+      expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
+      expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
+      expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
+    });
+  });
+};
+
+/**
+ * Run tests for AutomergeStorageAdapter.
+ */
+runTests('AutomergeStorageAdapter', () => {
+  const storage = createStorage({ type: StorageType.RAM });
+  afterTest(() => storage.close());
+  const dir = storage.createDirectory('automerge');
+  const adapter = new AutomergeStorageAdapter(dir);
+  afterTest(() => adapter.close());
+  return adapter;
+});
+
+/**
+ * Run tests for LevelDBStorageAdapter.
+ */
+runTests('LevelDBStorageAdapter', async () => {
+  const level = createTestLevel();
+  const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
+  await openAndClose(level, adapter as any);
+  return adapter;
+});

package/src/automerge/types.ts

@@ -2,11 +2,10 @@
 // Copyright 2024 DXOS.org
 //
 
-//
-// Copyright 2023 DXOS.org
-//
+import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
+import { type Level } from 'level';
 
-import { type Reference } from '@dxos/echo-db';
+import { type EncodedReferenceObject } from './reference';
 
 export type SpaceState = {
   // Url of the root automerge document.
@@ -79,5 +78,9 @@ export type ObjectSystem = {
   /**
    * Object reference ('protobuf' protocol) type.
    */
-  type?: Reference;
+  type?: EncodedReferenceObject;
 };
+
+export type LevelDB = Level<string, string>;
+export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
+export type BatchLevel = AbstractChainedBatch<any, string, string>;

package/src/db-host/data-service.ts

@@ -33,7 +33,7 @@ export class DataServiceImpl implements DataService {
   }
 
   async flush(request: FlushRequest): Promise<void> {
-    // TODO(dmaretskyi): Implement with automerge.
+    await this._automergeHost.flush(request);
   }
 
   // Automerge specific.

package/src/metadata/metadata-store.ts

@@ -11,7 +11,7 @@ import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { DataCorruptionError, STORAGE_VERSION, schema } from '@dxos/protocols';
-import { type Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
+import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import {
   type ControlPipelineSnapshot,
   EchoMetadata,
@@ -174,13 +174,8 @@ export class MetadataStore {
     scheduleTaskInterval(
       this._invitationCleanupCtx,
       async () => {
-        for (const invitation of this.getInvitations()) {
-          if (
-            invitation.created &&
-            invitation.lifetime &&
-            invitation.lifetime !== 0 &&
-            invitation.created.getTime() + invitation.lifetime * 1000 < Date.now()
-          ) {
+        for (const invitation of this._metadata.invitations ?? []) {
+          if (hasInvitationExpired(invitation) || isLegacyInvitationFormat(invitation)) {
             await this.removeInvitation(invitation.invitationId);
           }
         }
@@ -343,3 +338,17 @@ export class MetadataStore {
 }
 
 const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);
+
+export const hasInvitationExpired = (invitation: Invitation): boolean => {
+  return Boolean(
+    invitation.created &&
+      invitation.lifetime &&
+      invitation.lifetime !== 0 &&
+      invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
+  );
+};
+
+// TODO: remove once "multiuse" type invitations get removed from local metadata of existing profiles
+const isLegacyInvitationFormat = (invitation: Invitation): boolean => {
+  return invitation.type === Invitation.Type.MULTIUSE;
+};

package/src/space/control-pipeline.ts

@@ -4,7 +4,13 @@
 
 import { DeferredTask, sleepWithContext, trackLeaks } from '@dxos/async';
 import { Context } from '@dxos/context';
-import { SpaceStateMachine, type SpaceState, type MemberInfo, type FeedInfo } from '@dxos/credentials';
+import {
+  SpaceStateMachine,
+  type SpaceState,
+  type MemberInfo,
+  type FeedInfo,
+  type DelegateInvitationCredential,
+} from '@dxos/credentials';
 import { type FeedWrapper } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
@@ -50,6 +56,8 @@ export class ControlPipeline {
   public readonly onFeedAdmitted = new Callback<AsyncCallback<FeedInfo>>();
   public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
   public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
+  public readonly onDelegatedInvitation: Callback<AsyncCallback<DelegateInvitationCredential>>;
+  public readonly onDelegatedInvitationRemoved: Callback<AsyncCallback<DelegateInvitationCredential>>;
 
   @trace.metricsCounter()
   private _usage = new TimeUsageCounter();
@@ -92,6 +100,8 @@ export class ControlPipeline {
 
     this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
     this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
+    this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
+    this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;
   }
 
   get spaceState(): SpaceState {

package/src/space/space-manager.ts

@@ -3,6 +3,7 @@
 //
 
 import { synchronized, trackLeaks } from '@dxos/async';
+import { type DelegateInvitationCredential } from '@dxos/credentials';
 import { failUndefined } from '@dxos/debug';
 import { type FeedStore } from '@dxos/feed-store';
 import { PublicKey } from '@dxos/keys';
@@ -42,6 +43,7 @@ export type ConstructSpaceParams = {
    */
   onAuthorizedConnection: (session: Teleport) => void;
   onAuthFailure?: (session: Teleport) => void;
+  onDelegatedInvitationStatusChange: (invitation: DelegateInvitationCredential, isActive: boolean) => Promise<void>;
 };
 
 /**
@@ -84,6 +86,7 @@ export class SpaceManager {
     swarmIdentity,
     onAuthorizedConnection,
     onAuthFailure,
+    onDelegatedInvitationStatusChange,
     memberKey,
   }: ConstructSpaceParams) {
     log.trace('dxos.echo.space-manager.construct-space', trace.begin({ id: this._instanceId }));
@@ -111,6 +114,7 @@ export class SpaceManager {
       metadataStore: this._metadataStore,
       snapshotManager,
       memberKey,
+      onDelegatedInvitationStatusChange,
     });
     this._spaces.set(space.key, space);
 

package/src/space/space.test.ts

@@ -20,7 +20,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space = await agent.createSpace();
 
-    await space.open(new Context());
+    await space.open(Context.default());
     expect(space.isOpen).toBeTruthy();
     afterTest(() => space.close());
 
@@ -43,7 +43,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey);
 
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -62,7 +62,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, undefined, true);
 
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -114,7 +114,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space1 = await agent.createSpace();
 
-    await space1.open(new Context());
+    await space1.open(Context.default());
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -128,7 +128,7 @@ describe('space/space', () => {
     // Re-open.
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
-    await space2.open(new Context());
+    await space2.open(Context.default());
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
   });
 
@@ -139,7 +139,7 @@ describe('space/space', () => {
     const space = await agent.createSpace();
 
     {
-      await space.open(new Context());
+      await space.open(Context.default());
       afterTest(() => space.close());
       expect(space.isOpen).toBeTruthy();
 
@@ -153,7 +153,7 @@ describe('space/space', () => {
 
     // Re-open.
     {
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
 
       await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);