@dxos/echo-pipeline 0.5.0 → 0.5.1-main.0ba1ecb

package/src/automerge/echo-replicator.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Message } from '@dxos/automerge/automerge-repo';
+
+export interface EchoReplicator {
+  /**
+   * Called on when replicator is added to EchoHost.
+   */
+  connect(context: EchoReplicatorContext): Promise<void>;
+
+  /**
+   * Called on when replicator is removed from EchoHost.
+   */
+  disconnect(): Promise<void>;
+}
+
+export interface EchoReplicatorContext {
+  /**
+   * Our own peer id.
+   */
+  get peerId(): string;
+
+  onConnectionOpen(connection: ReplicatorConnection): void;
+
+  onConnectionClosed(connection: ReplicatorConnection): void;
+}
+
+export interface ReplicatorConnection {
+  /**
+   * Remove peer id.
+   */
+  get peerId(): string;
+
+  /**
+   * Stream to read messages coming from the remote peer.
+   */
+  readable: ReadableStream<Message>;
+
+  /**
+   * Stream to write messages to the remote peer.
+   */
+  writable: WritableStream<Message>;
+
+  /**
+   * @returns true if the document should be advertized to this peer.
+   *
+   * The remote peer can still request the document by it's id bypassing this check.
+   */
+  shouldAdvertize(params: ShouldAdvertizeParams): Promise<boolean>;
+}
+
+export type ShouldAdvertizeParams = {
+  documentId: string;
+};

package/src/automerge/index.ts

@@ -8,5 +8,4 @@ export * from './automerge-doc-loader';
 export * from './leveldb-storage-adapter';
 export * from './local-host-network-adapter';
 export * from './mesh-network-adapter';
-export * from './types';
-export * from './reference';
+export * from './echo-replicator';

package/src/automerge/leveldb-storage-adapter.ts

@@ -6,10 +6,9 @@ import { type MixedEncoding } from 'level-transcoder';
 
 import { type StorageAdapterInterface, type Chunk, type StorageKey } from '@dxos/automerge/automerge-repo';
 import { LifecycleState, Resource } from '@dxos/context';
+import { type BatchLevel, type SubLevelDB } from '@dxos/kv-store';
 import { type MaybePromise } from '@dxos/util';
 
-import { type BatchLevel, type SubLevelDB } from './types';
-
 export type LevelDBStorageAdapterParams = {
   db: SubLevelDB;
   callbacks?: StorageCallbacks;
@@ -107,6 +106,7 @@ const keyEncoder: MixedEncoding<StorageKey, Uint8Array, StorageKey> = {
       .toString()
       .split('-')
       .map((k) => k.replaceAll('%2D', '-').replaceAll('%25', '%')),
+  format: 'buffer',
 };
 
 export const encodingOptions = {

package/src/automerge/migrations.ts

@@ -4,12 +4,12 @@
 
 import { type StorageKey } from '@dxos/automerge/automerge-repo';
 import { IndexedDBStorageAdapter } from '@dxos/automerge/automerge-repo-storage-indexeddb';
+import { type SubLevelDB } from '@dxos/kv-store';
 import { log } from '@dxos/log';
 import { StorageType, type Directory } from '@dxos/random-access-storage';
 
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { encodingOptions } from './leveldb-storage-adapter';
-import { type SubLevelDB } from './types';
 
 export const levelMigration = async ({ db, directory }: { db: SubLevelDB; directory: Directory }) => {
   // Note: Make automigration from previous storage to leveldb here.

package/src/automerge/storage-adapter.test.ts

@@ -6,13 +6,13 @@ import { expect } from 'chai';
 
 import { type StorageAdapterInterface } from '@dxos/automerge/automerge-repo';
 import { PublicKey } from '@dxos/keys';
+import { createTestLevel } from '@dxos/kv-store/testing';
 import { StorageType, createStorage } from '@dxos/random-access-storage';
 import { afterTest, describe, openAndClose, test } from '@dxos/test';
 import { type MaybePromise } from '@dxos/util';
 
 import { AutomergeStorageAdapter } from './automerge-storage-adapter';
 import { LevelDBStorageAdapter } from './leveldb-storage-adapter';
-import { createTestLevel } from '../testing';
 
 const runTests = (
   testNamespace: string,

package/src/pipeline/pipeline.ts

@@ -172,6 +172,7 @@ export class PipelineState {
   }
 }
 
+// TODO(mykola): Extract to `@dxos/echo-protocol`
 export interface PipelineAccessor {
   state: PipelineState;
   writer: FeedWriter<FeedMessage.Payload>;

package/src/space/control-pipeline.ts

@@ -54,7 +54,7 @@ export class ControlPipeline {
   private _lastTimeframeSaveTime: number = Date.now();
 
   public readonly onFeedAdmitted = new Callback<AsyncCallback<FeedInfo>>();
-  public readonly onMemberAdmitted: Callback<AsyncCallback<MemberInfo>>;
+  public readonly onMemberRoleChanged: Callback<AsyncCallback<MemberInfo[]>>;
   public readonly onCredentialProcessed: Callback<AsyncCallback<Credential>>;
   public readonly onDelegatedInvitation: Callback<AsyncCallback<DelegateInvitationCredential>>;
   public readonly onDelegatedInvitationRemoved: Callback<AsyncCallback<DelegateInvitationCredential>>;
@@ -98,7 +98,7 @@ export class ControlPipeline {
       await this.onFeedAdmitted.callIfSet(info);
     });
 
-    this.onMemberAdmitted = this._spaceStateMachine.onMemberAdmitted;
+    this.onMemberRoleChanged = this._spaceStateMachine.onMemberRoleChanged;
     this.onCredentialProcessed = this._spaceStateMachine.onCredentialProcessed;
     this.onDelegatedInvitation = this._spaceStateMachine.onDelegatedInvitation;
     this.onDelegatedInvitationRemoved = this._spaceStateMachine.onDelegatedInvitationRemoved;

package/src/testing/index.ts

@@ -5,4 +5,3 @@
 export * from './change-metadata';
 export * from './test-agent-builder';
 export * from './test-feed-builder';
-export * from './level';

package/dist/types/src/automerge/level.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=level.test.d.ts.map

package/dist/types/src/automerge/level.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"level.test.d.ts","sourceRoot":"","sources":["../../../../src/automerge/level.test.ts"],"names":[],"mappings":""}

package/dist/types/src/automerge/reference.d.ts

@@ -1,15 +0,0 @@
-import { Reference } from '@dxos/echo-schema';
-export declare const REFERENCE_TYPE_TAG = "dxos.echo.model.document.Reference";
-/**
- * Reference as it is stored in Automerge document.
- */
-export type EncodedReferenceObject = {
-    '@type': typeof REFERENCE_TYPE_TAG;
-    itemId: string | null;
-    protocol: string | null;
-    host: string | null;
-};
-export declare const encodeReference: (reference: Reference) => EncodedReferenceObject;
-export declare const decodeReference: (value: any) => Reference;
-export declare const isEncodedReferenceObject: (value: any) => value is EncodedReferenceObject;
-//# sourceMappingURL=reference.d.ts.map

package/dist/types/src/automerge/reference.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"reference.d.ts","sourceRoot":"","sources":["../../../../src/automerge/reference.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9C,eAAO,MAAM,kBAAkB,uCAAuC,CAAC;AAEvE;;GAEG;AACH,MAAM,MAAM,sBAAsB,GAAG;IACnC,OAAO,EAAE,OAAO,kBAAkB,CAAC;IACnC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;CACrB,CAAC;AAEF,eAAO,MAAM,eAAe,cAAe,SAAS,KAAG,sBAMrD,CAAC;AAEH,eAAO,MAAM,eAAe,UAAW,GAAG,cACyC,CAAC;AAEpF,eAAO,MAAM,wBAAwB,UAAW,GAAG,oCACmC,CAAC"}

package/dist/types/src/automerge/types.d.ts

@@ -1,73 +0,0 @@
-/// <reference types="node" />
-import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
-import { type Level } from 'level';
-import { type EncodedReferenceObject } from './reference';
-export type SpaceState = {
-    rootUrl?: string;
-};
-export interface SpaceDoc {
-    access?: {
-        spaceKey: string;
-    };
-    /**
-     * Objects inlined in the current document.
-     */
-    objects?: {
-        [key: string]: ObjectStructure;
-    };
-    /**
-     * Object id points to an automerge doc url where the object is embedded.
-     */
-    links?: {
-        [echoId: string]: string;
-    };
-}
-/**
- * Representation of an ECHO object in an AM document.
- */
-export type ObjectStructure = {
-    data: Record<string, any>;
-    meta: ObjectMeta;
-    system: ObjectSystem;
-};
-/**
- * Echo object metadata.
- */
-export type ObjectMeta = {
-    /**
-     * Foreign keys.
-     */
-    keys: ForeignKey[];
-};
-/**
- * Reference to an object in a foreign database.
- */
-export type ForeignKey = {
-    /**
-     * Name of the foreign database/system.
-     * E.g. `github.com`.
-     */
-    source?: string;
-    /**
-     * Id within the foreign database.
-     */
-    id?: string;
-};
-/**
- * Automerge object system properties.
- * (Is automerge specific.)
- */
-export type ObjectSystem = {
-    /**
-     * Deletion marker.
-     */
-    deleted?: boolean;
-    /**
-     * Object reference ('protobuf' protocol) type.
-     */
-    type?: EncodedReferenceObject;
-};
-export type LevelDB = Level<string, string>;
-export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
-export type BatchLevel = AbstractChainedBatch<any, string, string>;
-//# sourceMappingURL=types.d.ts.map

package/dist/types/src/automerge/types.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/automerge/types.ts"],"names":[],"mappings":";AAIA,OAAO,EAAE,KAAK,oBAAoB,EAAE,KAAK,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAClF,OAAO,EAAE,KAAK,KAAK,EAAE,MAAM,OAAO,CAAC;AAEnC,OAAO,EAAE,KAAK,sBAAsB,EAAE,MAAM,aAAa,CAAC;AAE1D,MAAM,MAAM,UAAU,GAAG;IAEvB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CAAC;AAEF,MAAM,WAAW,QAAQ;IACvB,MAAM,CAAC,EAAE;QACP,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF;;OAEG;IACH,OAAO,CAAC,EAAE;QACR,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAAC;KAChC,CAAC;IACF;;OAEG;IACH,KAAK,CAAC,EAAE;QACN,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC;KAC1B,CAAC;CACH;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,YAAY,CAAC;CACtB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB;;OAEG;IACH,IAAI,EAAE,UAAU,EAAE,CAAC;CACpB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG;IACvB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;OAEG;IACH,EAAE,CAAC,EAAE,MAAM,CAAC;CACb,CAAC;AAEF;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG;IACzB;;OAEG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;OAEG;IACH,IAAI,CAAC,EAAE,sBAAsB,CAAC;CAC/B,CAAC;AAEF,MAAM,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAC5C,MAAM,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,UAAU,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAC7F,MAAM,MAAM,UAAU,GAAG,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC"}

package/dist/types/src/testing/level.d.ts

@@ -1,3 +0,0 @@
-import { type LevelDB } from '../automerge/types';
-export declare const createTestLevel: () => LevelDB;
-//# sourceMappingURL=level.d.ts.map

package/dist/types/src/testing/level.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"level.d.ts","sourceRoot":"","sources":["../../../../src/testing/level.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,KAAK,OAAO,EAAE,MAAM,oBAAoB,CAAC;AAElD,eAAO,MAAM,eAAe,QAAO,OAA+E,CAAC"}

package/src/automerge/level.test.ts

@@ -1,82 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { expect } from 'chai';
-import { Level } from 'level';
-
-import { PublicKey } from '@dxos/keys';
-import { describe, openAndClose, test } from '@dxos/test';
-
-import { type SubLevelDB } from './types';
-import { createTestLevel } from '../testing';
-
-describe('Level', () => {
-  test('missing keys', async () => {
-    const level = createTestLevel();
-    await openAndClose(level);
-
-    expect(() => level.get('missing')).to.throw;
-  });
-
-  test('data persistance after reload', async () => {
-    const path = `/tmp/dxos-${PublicKey.random().toHex()}`;
-    const level = new Level<string, string>(path);
-    await level.open();
-
-    const key = 'name';
-    const value = 'Rich';
-    {
-      await level.put(key, value);
-      expect(await level.get(key)).to.equal(value);
-    }
-
-    await level.close();
-
-    {
-      const level = new Level<string, string>(path);
-      await level.open();
-      expect(await level.get(key)).to.equal(value);
-      await level.clear();
-      expect(() => level.get(key)).to.throw;
-    }
-  });
-
-  test('batch different sublevels', async () => {
-    const level = createTestLevel();
-    await openAndClose(level);
-
-    const first: SubLevelDB = level.sublevel('first');
-    const second: SubLevelDB = level.sublevel('second');
-
-    const batch = first.batch();
-
-    const key = 'key';
-    const value = 'first-level-value';
-    batch.put(key, value, { sublevel: second });
-    await batch.write();
-
-    expect(() => first.get(key)).to.throw;
-    expect(await level.sublevel('second').get(key)).to.equal(value);
-    expect(await second.get(key)).to.equal(value);
-    expect(await second.iterator().all()).to.deep.equal([[key, value]]);
-  });
-
-  test('sublevel prefixes', () => {
-    const level = createTestLevel();
-    expect(level.prefixKey('', 'utf8')).to.equal('');
-
-    {
-      const sublevel = level.sublevel('sublevel');
-      expect(sublevel.prefixKey('', 'utf8')).to.equal('!sublevel!');
-    }
-
-    {
-      const sublevel = level.sublevel('');
-      expect(sublevel.prefixKey('', 'utf8')).to.equal('!!');
-
-      const sublevel2 = sublevel.sublevel('');
-      expect(sublevel2.prefixKey('', 'utf8')).to.equal('!!!!');
-    }
-  });
-});

package/src/automerge/reference.ts

@@ -1,31 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { Reference } from '@dxos/echo-schema';
-
-export const REFERENCE_TYPE_TAG = 'dxos.echo.model.document.Reference';
-
-/**
- * Reference as it is stored in Automerge document.
- */
-export type EncodedReferenceObject = {
-  '@type': typeof REFERENCE_TYPE_TAG;
-  itemId: string | null;
-  protocol: string | null;
-  host: string | null;
-};
-
-export const encodeReference = (reference: Reference): EncodedReferenceObject => ({
-  '@type': REFERENCE_TYPE_TAG,
-  // NOTE: Automerge do not support undefined values, so we need to use null instead.
-  itemId: reference.itemId ?? null,
-  protocol: reference.protocol ?? null,
-  host: reference.host ?? null,
-});
-
-export const decodeReference = (value: any) =>
-  new Reference(value.itemId, value.protocol ?? undefined, value.host ?? undefined);
-
-export const isEncodedReferenceObject = (value: any): value is EncodedReferenceObject =>
-  typeof value === 'object' && value !== null && value['@type'] === REFERENCE_TYPE_TAG;

package/src/automerge/types.ts

@@ -1,86 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { type AbstractChainedBatch, type AbstractSublevel } from 'abstract-level';
-import { type Level } from 'level';
-
-import { type EncodedReferenceObject } from './reference';
-
-export type SpaceState = {
-  // Url of the root automerge document.
-  rootUrl?: string;
-};
-
-export interface SpaceDoc {
-  access?: {
-    spaceKey: string;
-  };
-  /**
-   * Objects inlined in the current document.
-   */
-  objects?: {
-    [key: string]: ObjectStructure;
-  };
-  /**
-   * Object id points to an automerge doc url where the object is embedded.
-   */
-  links?: {
-    [echoId: string]: string;
-  };
-}
-
-/**
- * Representation of an ECHO object in an AM document.
- */
-export type ObjectStructure = {
-  data: Record<string, any>;
-  meta: ObjectMeta;
-  system: ObjectSystem;
-};
-
-/**
- * Echo object metadata.
- */
-export type ObjectMeta = {
-  /**
-   * Foreign keys.
-   */
-  keys: ForeignKey[];
-};
-
-/**
- * Reference to an object in a foreign database.
- */
-export type ForeignKey = {
-  /**
-   * Name of the foreign database/system.
-   * E.g. `github.com`.
-   */
-  source?: string;
-
-  /**
-   * Id within the foreign database.
-   */
-  id?: string;
-};
-
-/**
- * Automerge object system properties.
- * (Is automerge specific.)
- */
-export type ObjectSystem = {
-  /**
-   * Deletion marker.
-   */
-  deleted?: boolean;
-
-  /**
-   * Object reference ('protobuf' protocol) type.
-   */
-  type?: EncodedReferenceObject;
-};
-
-export type LevelDB = Level<string, string>;
-export type SubLevelDB = AbstractSublevel<any, string | Buffer | Uint8Array, string, string>;
-export type BatchLevel = AbstractChainedBatch<any, string, string>;

package/src/testing/level.ts

@@ -1,11 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { Level } from 'level';
-
-import { PublicKey } from '@dxos/keys';
-
-import { type LevelDB } from '../automerge/types';
-
-export const createTestLevel = (): LevelDB => new Level<string, string>(`/tmp/dxos-${PublicKey.random().toHex()}`);