@dxos/echo-pipeline 0.4.9 → 0.4.10-main.068c3d8

package/src/automerge/storage-adapter.test.ts

@@ -0,0 +1,90 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { expect } from 'chai';
+
+import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
+import { PublicKey } from '@dxos/keys';
+import { StorageType, createStorage } from '@dxos/random-access-storage';
+import { afterTest, describe, openAndClose, test } from '@dxos/test';
+import { type MaybePromise } from '@dxos/util';
+
+import { AutomergeStorageAdapter } from './automerge-storage-adapter';
+import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
+import { createTestLevel } from '../testing';
+
+const runTests = (
+  testNamespace: string,
+  /** Run per test. Expects automatic clean-up with `afterTest`. */ createAdapter: () => MaybePromise<StorageAdapterInterface>,
+) => {
+  describe(testNamespace, () => {
+    const chunks = [
+      { key: ['a', 'b', 'c', '1'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'c', '2'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '3'], data: PublicKey.random().asUint8Array() },
+      { key: ['a', 'b', 'd', '4'], data: PublicKey.random().asUint8Array() },
+    ];
+
+    test('should store and retrieve data', async () => {
+      const adapter = await createAdapter();
+
+      await adapter.save(chunks[0].key, chunks[0].data);
+      expect(await adapter.load(chunks[0].key)).to.deep.equal(chunks[0].data);
+    });
+
+    test('loadRange return inputs with correct prefixes', async () => {
+      const adapter = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(4);
+      expect((await adapter.loadRange(['a', 'b', 'c']))[0]).to.deep.equal(chunks[0]);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(2);
+    });
+
+    test('deletion works', async () => {
+      const adapter = await createAdapter();
+
+      for (const chunk of chunks) {
+        await adapter.save(chunk.key, chunk.data);
+      }
+
+      await adapter.remove(['a', 'b', 'c', '1']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(3);
+      expect((await adapter.loadRange(['a', 'b', 'c'])).length).to.equal(1);
+
+      await adapter.removeRange(['a', 'b', 'd']);
+
+      expect((await adapter.loadRange(['a', 'b'])).length).to.equal(1);
+      expect((await adapter.loadRange(['a', 'b']))[0]).to.deep.equal(chunks[1]);
+      expect(await adapter.load(['a', 'b', 'c', '2'])).to.deep.equal(chunks[1].data);
+      expect(await adapter.load(['a', 'b', 'd', '3'])).to.be.undefined;
+    });
+  });
+};
+
+/**
+ * Run tests for AutomergeStorageAdapter.
+ */
+runTests('AutomergeStorageAdapter', () => {
+  const storage = createStorage({ type: StorageType.RAM });
+  afterTest(() => storage.close());
+  const dir = storage.createDirectory('automerge');
+  const adapter = new AutomergeStorageAdapter(dir);
+  afterTest(() => adapter.close());
+  return adapter;
+});
+
+/**
+ * Run tests for LevelDBStorageAdapter.
+ */
+runTests('LevelDBStorageAdapter', async () => {
+  const level = createTestLevel();
+  const adapter = new LevelDBStorageAdapter({ db: level.sublevel('automerge') });
+  await openAndClose(level, adapter as any);
+  return adapter;
+});

package/src/automerge/types.ts

@@ -0,0 +1,86 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
+import { type Level } from 'level';
+
+import { type EncodedReferenceObject } from './reference';
+
+export type SpaceState = {
+  // Url of the root automerge document.
+  rootUrl?: string;
+};
+
+export interface SpaceDoc {
+  access?: {
+    spaceKey: string;
+  };
+  /**
+   * Objects inlined in the current document.
+   */
+  objects?: {
+    [key: string]: ObjectStructure;
+  };
+  /**
+   * Object id points to an automerge doc url where the object is embedded.
+   */
+  links?: {
+    [echoId: string]: string;
+  };
+}
+
+/**
+ * Representation of an ECHO object in an AM document.
+ */
+export type ObjectStructure = {
+  data: Record<string, any>;
+  meta: ObjectMeta;
+  system: ObjectSystem;
+};
+
+/**
+ * Echo object metadata.
+ */
+export type ObjectMeta = {
+  /**
+   * Foreign keys.
+   */
+  keys: ForeignKey[];
+};
+
+/**
+ * Reference to an object in a foreign database.
+ */
+export type ForeignKey = {
+  /**
+   * Name of the foreign database/system.
+   * E.g. `github.com`.
+   */
+  source?: string;
+
+  /**
+   * Id within the foreign database.
+   */
+  id?: string;
+};
+
+/**
+ * Automerge object system properties.
+ * (Is automerge specific.)
+ */
+export type ObjectSystem = {
+  /**
+   * Deletion marker.
+   */
+  deleted?: boolean;
+
+  /**
+   * Object reference ('protobuf' protocol) type.
+   */
+  type?: EncodedReferenceObject;
+};
+
+export type LevelDB = Level<string, string>;
+export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
+export type BatchLevel = AbstractChainedBatch<any, string, string>;

package/src/db-host/data-service.ts

@@ -33,7 +33,7 @@ export class DataServiceImpl implements DataService {
   }
 
   async flush(request: FlushRequest): Promise<void> {
-    // TODO(dmaretskyi): Implement with automerge.
+    await this._automergeHost.flush(request);
   }
 
   // Automerge specific.

package/src/metadata/metadata-store.ts

@@ -11,7 +11,7 @@ import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { DataCorruptionError, STORAGE_VERSION, schema } from '@dxos/protocols';
-import { type Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
+import { Invitation, SpaceState } from '@dxos/protocols/proto/dxos/client/services';
 import {
   type ControlPipelineSnapshot,
   EchoMetadata,
@@ -174,13 +174,8 @@ export class MetadataStore {
     scheduleTaskInterval(
       this._invitationCleanupCtx,
       async () => {
-        for (const invitation of this.getInvitations()) {
-          if (
-            invitation.created &&
-            invitation.lifetime &&
-            invitation.lifetime !== 0 &&
-            invitation.created.getTime() + invitation.lifetime * 1000 < Date.now()
-          ) {
+        for (const invitation of this._metadata.invitations ?? []) {
+          if (hasInvitationExpired(invitation) || isLegacyInvitationFormat(invitation)) {
             await this.removeInvitation(invitation.invitationId);
           }
         }
@@ -343,3 +338,17 @@ export class MetadataStore {
 }
 
 const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);
+
+export const hasInvitationExpired = (invitation: Invitation): boolean => {
+  return Boolean(
+    invitation.created &&
+      invitation.lifetime &&
+      invitation.lifetime !== 0 &&
+      invitation.created.getTime() + invitation.lifetime * 1000 < Date.now(),
+  );
+};
+
+// TODO: remove once "multiuse" type invitations get removed from local metadata of existing profiles
+const isLegacyInvitationFormat = (invitation: Invitation): boolean => {
+  return invitation.type === Invitation.Type.MULTIUSE;
+};

package/src/space/space.test.ts

@@ -20,7 +20,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space = await agent.createSpace();
 
-    await space.open(new Context());
+    await space.open(Context.default());
     expect(space.isOpen).toBeTruthy();
     afterTest(() => space.close());
 
@@ -43,7 +43,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey);
 
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -62,7 +62,7 @@ describe('space/space', () => {
       const agent = await builder.createPeer();
       const space = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, undefined, true);
 
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
       afterTest(() => space.close());
 
@@ -114,7 +114,7 @@ describe('space/space', () => {
     const agent = await builder.createPeer();
     const space1 = await agent.createSpace();
 
-    await space1.open(new Context());
+    await space1.open(Context.default());
     expect(space1.isOpen).toBeTruthy();
     afterTest(() => space1.close());
 
@@ -128,7 +128,7 @@ describe('space/space', () => {
     // Re-open.
     const space2 = await agent.createSpace(agent.identityKey, space1.key, space1.genesisFeedKey, space1.dataFeedKey);
 
-    await space2.open(new Context());
+    await space2.open(Context.default());
     await space2.controlPipeline.state!.waitUntilTimeframe(space2.controlPipeline.state!.endTimeframe);
   });
 
@@ -139,7 +139,7 @@ describe('space/space', () => {
     const space = await agent.createSpace();
 
     {
-      await space.open(new Context());
+      await space.open(Context.default());
       afterTest(() => space.close());
       expect(space.isOpen).toBeTruthy();
 
@@ -153,7 +153,7 @@ describe('space/space', () => {
 
     // Re-open.
     {
-      await space.open(new Context());
+      await space.open(Context.default());
       expect(space.isOpen).toBeTruthy();
 
       await space.controlPipeline.state!.waitUntilTimeframe(space.controlPipeline.state!.endTimeframe);

package/src/space/space.ts

@@ -3,7 +3,7 @@
 //
 
 import { Event, Mutex, synchronized, trackLeaks } from '@dxos/async';
-import { type Context } from '@dxos/context';
+import { Resource, type Context, LifecycleState } from '@dxos/context';
 import { type FeedInfo } from '@dxos/credentials';
 import { type FeedOptions, type FeedWrapper } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
@@ -48,7 +48,7 @@ export type CreatePipelineParams = {
 // TODO(dmaretskyi): Extract database stuff.
 @trackLeaks('open', 'close')
 @trace.resource()
-export class Space {
+export class Space extends Resource {
   private readonly _addFeedMutex = new Mutex();
 
   public readonly onCredentialProcessed = new Callback<AsyncCallback<Credential>>();
@@ -64,11 +64,11 @@ export class Space {
 
   private readonly _snapshotManager: SnapshotManager;
 
-  private _isOpen = false;
   private _controlFeed?: FeedWrapper<FeedMessage>;
   private _dataFeed?: FeedWrapper<FeedMessage>;
 
   constructor(params: SpaceParams) {
+    super();
     invariant(params.spaceKey && params.feedProvider);
     this._key = params.spaceKey;
     this._genesisFeedKey = params.genesisFeed.key;
@@ -112,7 +112,7 @@ export class Space {
   }
 
   get isOpen() {
-    return this._isOpen;
+    return this._lifecycleState === LifecycleState.OPEN;
   }
 
   get genesisFeedKey(): PublicKey {
@@ -162,40 +162,25 @@ export class Space {
     return Array.from(this._controlPipeline.spaceState.feeds.values());
   }
 
-  /**
-   * Use for diagnostics.
-   */
-  // getDataFeeds(): FeedInfo[] {
-  //   return this._dataPipeline?.getFeeds();
-  // }
-  @synchronized
   @trace.span()
-  async open(ctx: Context) {
+  protected override async _open(ctx: Context) {
     log('opening...');
-    if (this._isOpen) {
-      return;
-    }
 
     // Order is important.
     await this._controlPipeline.start();
     await this.protocol.start();
 
-    this._isOpen = true;
     log('opened');
   }
 
   @synchronized
-  async close() {
+  protected override async _close() {
     log('closing...', { key: this._key });
-    if (!this._isOpen) {
-      return;
-    }
 
     // Closes in reverse order to open.
     await this.protocol.stop();
     await this._controlPipeline.stop();
 
-    this._isOpen = false;
     log('closed');
   }
 }

package/src/testing/index.ts

@@ -5,3 +5,4 @@
 export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
+export * from './level';

package/src/testing/level.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Level } from 'level';
+
+import { PublicKey } from '@dxos/keys';
+
+import { type LevelDB } from '../automerge/types';
+
+export const createTestLevel = (): LevelDB => new Level<string, string>(`/tmp/dxos-${PublicKey.random().toHex()}`);