@dxos/echo-pipeline 0.1.24-next.6b1f434

package/dist/types/src/testing/test-feed-builder.d.ts

@@ -0,0 +1,9 @@
+import { TestBuilder } from '@dxos/feed-store/testing';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+/**
+ * Builder with default encoder and generator.
+ */
+export declare class TestFeedBuilder extends TestBuilder<FeedMessage> {
+    constructor();
+}
+//# sourceMappingURL=test-feed-builder.d.ts.map

package/dist/types/src/testing/test-feed-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"test-feed-builder.d.ts","sourceRoot":"","sources":["../../../../src/testing/test-feed-builder.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sCAAsC,CAAC;AAIxE;;GAEG;AACH,qBAAa,eAAgB,SAAQ,WAAW,CAAC,WAAW,CAAC;;CAM5D"}

package/dist/types/src/testing/util.d.ts

@@ -0,0 +1,14 @@
+import { DatabaseBackendProxy, ItemManager } from '@dxos/echo-db';
+import { ModelFactory } from '@dxos/model-factory';
+import { DatabaseBackendHost, DataServiceHost } from '../dbhost';
+import { DataPipelineControllerImpl } from '../space';
+export declare const createMemoryDatabase: (modelFactory: ModelFactory) => Promise<{
+    backend: DatabaseBackendHost;
+    itemManager: ItemManager;
+}>;
+export declare const createRemoteDatabaseFromDataServiceHost: (modelFactory: ModelFactory, dataServiceHost: DataServiceHost) => Promise<{
+    itemManager: ItemManager;
+    backend: DatabaseBackendProxy;
+}>;
+export declare const testLocalDatabase: (create: DataPipelineControllerImpl, check?: DataPipelineControllerImpl) => Promise<void>;
+//# sourceMappingURL=util.d.ts.map

package/dist/types/src/testing/util.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/testing/util.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAGlE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAInD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAA6C,MAAM,WAAW,CAAC;AAC5G,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,eAAO,MAAM,oBAAoB,iBAAwB,YAAY;;;EAqBpE,CAAC;AAEF,eAAO,MAAM,uCAAuC,iBACpC,YAAY,mBACT,eAAe;;;EAejC,CAAC;AAEF,eAAO,MAAM,iBAAiB,WACpB,0BAA0B,UAC3B,0BAA0B,kBAgBlC,CAAC"}

package/dist/types/src/tests/database.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=database.test.d.ts.map

package/dist/types/src/tests/database.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"database.test.d.ts","sourceRoot":"","sources":["../../../../src/tests/database.test.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,60 @@
+{
+  "name": "@dxos/echo-pipeline",
+  "version": "0.1.24-next.6b1f434",
+  "description": "ECHO database.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "license": "MIT",
+  "author": "info@dxos.org",
+  "main": "dist/lib/node/index.cjs",
+  "browser": {
+    "./dist/lib/node/index.cjs": "./dist/lib/browser/index.mjs",
+    "./testing.js": "./dist/lib/browser/testing/index.mjs"
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "testing.d.ts",
+    "testing.js",
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "debug": "^4.3.3",
+    "@dxos/async": "0.1.24-next.6b1f434",
+    "@dxos/codec-protobuf": "0.1.24-next.6b1f434",
+    "@dxos/context": "0.1.24-next.6b1f434",
+    "@dxos/credentials": "0.1.24-next.6b1f434",
+    "@dxos/crypto": "0.1.24-next.6b1f434",
+    "@dxos/debug": "0.1.24-next.6b1f434",
+    "@dxos/document-model": "0.1.24-next.6b1f434",
+    "@dxos/echo-db": "0.1.24-next.6b1f434",
+    "@dxos/feed-store": "0.1.24-next.6b1f434",
+    "@dxos/hypercore": "0.1.24-next.6b1f434",
+    "@dxos/keyring": "0.1.24-next.6b1f434",
+    "@dxos/keys": "0.1.24-next.6b1f434",
+    "@dxos/log": "0.1.24-next.6b1f434",
+    "@dxos/messaging": "0.1.24-next.6b1f434",
+    "@dxos/model-factory": "0.1.24-next.6b1f434",
+    "@dxos/network-manager": "0.1.24-next.6b1f434",
+    "@dxos/node-std": "0.1.24-next.6b1f434",
+    "@dxos/protocols": "0.1.24-next.6b1f434",
+    "@dxos/random-access-storage": "0.1.24-next.6b1f434",
+    "@dxos/rpc": "0.1.24-next.6b1f434",
+    "@dxos/teleport": "0.1.24-next.6b1f434",
+    "@dxos/teleport-extension-object-sync": "0.1.24-next.6b1f434",
+    "@dxos/teleport-extension-presence": "0.1.24-next.6b1f434",
+    "@dxos/teleport-extension-replicator": "0.1.24-next.6b1f434",
+    "@dxos/timeframe": "0.1.24-next.6b1f434",
+    "@dxos/typings": "0.1.24-next.6b1f434",
+    "@dxos/util": "0.1.24-next.6b1f434"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.7",
+    "hypercore-protocol": "^8.0.7",
+    "source-map-support": "^0.5.12",
+    "wait-for-expect": "^3.0.2"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/common/codec.ts

@@ -0,0 +1,18 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { Codec } from '@dxos/codec-protobuf';
+import { createCodecEncoding } from '@dxos/hypercore';
+import { schema } from '@dxos/protocols';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+
+/**
+ * Codec for feed messages.
+ */
+export const codec: Codec<FeedMessage> = schema.getCodecForType('dxos.echo.feed.FeedMessage');
+
+/**
+ * Value encoding used by feed store.
+ */
+export const valueEncoding = createCodecEncoding(codec);

package/src/common/feeds.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { FeedWriter } from '@dxos/feed-store';
+import { MaybePromise } from '@dxos/util';
+
+/**
+ * Maps the written arguments onto a different message type.
+ */
+export const createMappedFeedWriter = <Source extends {}, Target extends {}>(
+  mapper: (arg: Source) => MaybePromise<Target>,
+  writer: FeedWriter<Target>
+): FeedWriter<Source> => {
+  assert(mapper);
+  assert(writer);
+
+  return {
+    write: async (data: Source) => await writer.write(await mapper(data))
+  };
+};

package/src/common/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './codec';
+export * from './feeds';

package/src/dbhost/data-service-host.ts

@@ -0,0 +1,103 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { Stream } from '@dxos/codec-protobuf';
+import { tagMutationsInBatch, ItemDemuxer, ItemManager } from '@dxos/echo-db';
+import { FeedWriter } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { EchoEvent, MutationReceipt, WriteRequest } from '@dxos/protocols/proto/dxos/echo/service';
+import { ComplexMap } from '@dxos/util';
+
+/**
+ * Provides methods for DataService for a single space.
+ * A DataServiceRouter must be placed before it to route requests to different DataServiceHost instances based on space id.
+ */
+// TODO(burdon): Move to client-services.
+export class DataServiceHost {
+  private readonly _clientTagMap = new ComplexMap<[feedKey: PublicKey, seq: number], string>(
+    ([feedKey, seq]) => `${feedKey.toHex()}:${seq}`
+  );
+
+  constructor(
+    private readonly _itemManager: ItemManager,
+    private readonly _itemDemuxer: ItemDemuxer,
+    private readonly _writeStream?: FeedWriter<DataMessage>
+  ) {}
+
+  /**
+   * Real-time subscription to data objects in a space.
+   */
+  subscribe(): Stream<EchoEvent> {
+    return new Stream(({ next, ctx }) => {
+      // send current state
+      const objects = Array.from(this._itemManager.entities.values()).map((entity) => entity.createSnapshot());
+
+      next({
+        batch: {
+          objects
+        }
+      });
+
+      // subscribe to mutations
+
+      this._itemDemuxer.mutation.on(ctx, (mutation) => {
+        log('Object update', { mutation });
+
+        const clientTag = this._clientTagMap.get([mutation.meta.feedKey, mutation.meta.seq]);
+        // TODO(dmaretskyi): Memorąąy leak with _clientTagMap not getting cleared.
+
+        // Assign feed metadata
+        const batch = {
+          objects: [
+            {
+              ...mutation.data,
+              mutations: mutation.data.mutations?.map((m, mutationIdx) => ({
+                ...m,
+                meta: mutation.meta
+              })),
+              meta: mutation.meta
+            }
+          ]
+        };
+
+        // Assign client tag metadata
+        if (clientTag) {
+          tagMutationsInBatch(batch, clientTag);
+        }
+
+        next({
+          clientTag,
+          feedKey: mutation.meta.feedKey,
+          seq: mutation.meta.seq,
+          batch
+        });
+      });
+    });
+  }
+
+  async write(request: WriteRequest): Promise<MutationReceipt> {
+    assert(this._writeStream, 'Cannot write mutations in readonly mode');
+    assert(request.batch.objects?.length === 1, 'Only single object mutations are supported');
+
+    const receipt = await this._writeStream.write({
+      object: {
+        ...request.batch.objects[0],
+        mutations: request.batch.objects[0].mutations?.map((m) => ({
+          ...m,
+          meta: undefined
+        })),
+        meta: undefined
+      }
+    });
+    if (request.clientTag) {
+      this._clientTagMap.set([receipt.feedKey, receipt.seq], request.clientTag);
+    }
+
+    return receipt;
+  }
+}

package/src/dbhost/data-service.ts

@@ -0,0 +1,66 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { Stream } from '@dxos/codec-protobuf';
+import { raise } from '@dxos/debug';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import {
+  DataService,
+  MutationReceipt,
+  SubscribeRequest,
+  EchoEvent,
+  WriteRequest
+} from '@dxos/protocols/proto/dxos/echo/service';
+import { ComplexMap } from '@dxos/util';
+
+import { DataServiceHost } from './data-service-host';
+
+// TODO(burdon): Clear on close.
+export class DataServiceSubscriptions {
+  private readonly _spaces = new ComplexMap<PublicKey, DataServiceHost>(PublicKey.hash);
+
+  clear() {
+    this._spaces.clear();
+  }
+
+  registerSpace(spaceKey: PublicKey, host: DataServiceHost) {
+    log('Registering space', { spaceKey });
+    this._spaces.set(spaceKey, host);
+  }
+
+  unregisterSpace(spaceKey: PublicKey) {
+    log('Unregistering space', { spaceKey });
+    this._spaces.delete(spaceKey);
+  }
+
+  getDataService(spaceKey: PublicKey) {
+    return this._spaces.get(spaceKey);
+  }
+}
+
+/**
+ * Routes DataService requests to different DataServiceHost instances based on space id.
+ */
+// TODO(burdon): Move to client-services.
+export class DataServiceImpl implements DataService {
+  constructor(private readonly _subscriptions: DataServiceSubscriptions) {}
+
+  subscribe(request: SubscribeRequest): Stream<EchoEvent> {
+    assert(request.spaceKey);
+    const host =
+      this._subscriptions.getDataService(request.spaceKey) ?? raise(new Error(`space not found: ${request.spaceKey}`));
+    return host.subscribe();
+  }
+
+  write(request: WriteRequest): Promise<MutationReceipt> {
+    assert(request.spaceKey);
+    assert(request.batch);
+    const host =
+      this._subscriptions.getDataService(request.spaceKey) ?? raise(new Error(`space not found: ${request.spaceKey}`));
+    return host.write(request);
+  }
+}

package/src/dbhost/database-backend.ts

@@ -0,0 +1,64 @@
+//
+
+//
+// Copyright 2023 DXOS.org
+//
+
+import { EchoProcessor, ItemDemuxer, ItemDemuxerOptions, ItemManager } from '@dxos/echo-db';
+import { FeedWriter } from '@dxos/feed-store';
+import { ModelFactory } from '@dxos/model-factory';
+import { DataMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { EchoSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+
+import { DataServiceHost } from './data-service-host';
+
+/**
+ * Database backend that operates on two streams: read and write.
+ * Mutations are read from the incoming streams and applied to the ItemManager via ItemDemuxer.
+ * Write operations result in mutations being written to the outgoing stream.
+ */
+export class DatabaseBackendHost {
+  private _echoProcessor!: EchoProcessor;
+  private _itemManager!: ItemManager;
+  private _itemDemuxer!: ItemDemuxer;
+
+  constructor(
+    private readonly _outboundStream: FeedWriter<DataMessage> | undefined,
+    private readonly _snapshot?: EchoSnapshot,
+    private readonly _options: ItemDemuxerOptions = {} // TODO(burdon): Pass in factory instead?
+  ) {}
+
+  get isReadOnly(): boolean {
+    return !!this._outboundStream;
+  }
+
+  get echoProcessor() {
+    return this._echoProcessor;
+  }
+
+  async open(itemManager: ItemManager, modelFactory: ModelFactory) {
+    this._itemManager = itemManager;
+    this._itemManager._debugLabel = 'host';
+
+    this._itemDemuxer = new ItemDemuxer(itemManager, modelFactory, this._options);
+    this._echoProcessor = this._itemDemuxer.open();
+
+    if (this._snapshot) {
+      await this._itemDemuxer.restoreFromSnapshot(this._snapshot);
+    }
+  }
+
+  async close() {}
+
+  getWriteStream(): FeedWriter<DataMessage> | undefined {
+    return this._outboundStream;
+  }
+
+  createSnapshot() {
+    return this._itemDemuxer.createSnapshot();
+  }
+
+  createDataServiceHost() {
+    return new DataServiceHost(this._itemManager, this._itemDemuxer, this._outboundStream ?? undefined);
+  }
+}

package/src/dbhost/index.ts

@@ -0,0 +1,9 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+export * from './data-service-host';
+export * from './database-backend';
+export * from './snapshot-manager';
+export * from './snapshot-store';
+export * from './data-service';

package/src/dbhost/snapshot-manager.ts

@@ -0,0 +1,68 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { trackLeaks } from '@dxos/async';
+import { schema } from '@dxos/protocols';
+import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { DataObject } from '@dxos/protocols/proto/dxos/mesh/teleport/objectsync';
+import { ObjectSync } from '@dxos/teleport-extension-object-sync';
+
+import { SnapshotStore } from './snapshot-store';
+
+/**
+ * Snapshot manager for a specific space.
+ */
+@trackLeaks('open', 'close')
+export class SnapshotManager {
+  private readonly _objectSync: ObjectSync;
+
+  // prettier-ignore
+  constructor(
+    private readonly _snapshotStore: SnapshotStore
+  ) {
+    this._objectSync = new ObjectSync({
+      getObject: async (id: string) => {
+        const snapshot = await this._snapshotStore.loadSnapshot(id);
+        if (!snapshot) {
+          return undefined;
+        }
+        return {
+          id,
+          payload: schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').encodeAsAny(snapshot)
+        };
+      },
+      setObject: async (data: DataObject) => {
+        const snapshot = schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').decode(data.payload);
+        await this._snapshotStore.saveSnapshot(snapshot);
+      }
+    });
+  }
+
+  get objectSync() {
+    return this._objectSync;
+  }
+
+  async open() {
+    await this._objectSync.open();
+  }
+
+  async close() {
+    await this._objectSync.close();
+  }
+
+  async load(id: string): Promise<SpaceSnapshot> {
+    const local = await this._snapshotStore.loadSnapshot(id);
+    if (local) {
+      return local;
+    }
+
+    const remote = await this._objectSync.download(id);
+    return schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').decode(remote.payload);
+  }
+
+  async store(snapshot: SpaceSnapshot): Promise<string> {
+    const id = await this._snapshotStore.saveSnapshot(snapshot);
+    return id;
+  }
+}

package/src/dbhost/snapshot-store.test.ts

@@ -0,0 +1,30 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import expect from 'expect';
+
+import { PublicKey } from '@dxos/keys';
+import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { createStorage } from '@dxos/random-access-storage';
+import { Timeframe } from '@dxos/timeframe';
+
+import { SnapshotStore } from './snapshot-store';
+
+describe('SnapshotStore', () => {
+  it('should save and load snapshot', async () => {
+    const store = new SnapshotStore(createStorage().createDirectory('snapshots'));
+
+    const snapshot: SpaceSnapshot = {
+      spaceKey: PublicKey.random().asBuffer(),
+      timeframe: new Timeframe(),
+      database: {
+        items: []
+      }
+    };
+
+    const key = await store.saveSnapshot(snapshot);
+    const loaded = await store.loadSnapshot(key);
+    expect(loaded).toEqual(snapshot);
+  });
+});

package/src/dbhost/snapshot-store.ts

@@ -0,0 +1,45 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { subtleCrypto } from '@dxos/crypto';
+import { schema } from '@dxos/protocols';
+import { SpaceSnapshot } from '@dxos/protocols/proto/dxos/echo/snapshot';
+import { Directory } from '@dxos/random-access-storage';
+
+export class SnapshotStore {
+  // prettier-ignore
+  constructor(
+    private readonly _directory: Directory
+  ) {}
+
+  async saveSnapshot(snapshot: SpaceSnapshot): Promise<string> {
+    const encoded = schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').encode(snapshot);
+    const key = await subtleCrypto.digest('SHA-256', encoded);
+    const keyString = Buffer.from(key).toString('hex');
+
+    const file = await this._directory.getOrCreateFile(keyString);
+    try {
+      await file.write(0, Buffer.from(encoded));
+    } finally {
+      await file.close();
+    }
+
+    return keyString;
+  }
+
+  async loadSnapshot(key: string): Promise<SpaceSnapshot | undefined> {
+    const file = await this._directory.getOrCreateFile(key);
+    try {
+      const { size } = await file.stat();
+      if (size === 0) {
+        return undefined;
+      }
+
+      const buffer = await file.read(0, size);
+      return schema.getCodecForType('dxos.echo.snapshot.SpaceSnapshot').decode(buffer);
+    } finally {
+      await file.close();
+    }
+  }
+}

package/src/errors/errors.ts

@@ -0,0 +1,52 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { PublicKey } from '@dxos/keys';
+import { ItemID } from '@dxos/protocols';
+
+// TODO(burdon): Reconcile with @dxos/errors.
+
+export class DBError extends Error {
+  constructor(readonly code: string, message?: string, readonly context?: any) {
+    super(message ? `${code}: ${message}` : code.toString());
+    // NOTE: Restores prototype chain (https://stackoverflow.com/a/48342359).
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+export class IdentityNotInitializedError extends DBError {
+  constructor() {
+    super('IDENTITY_NOT_INITIALIZED', 'Identity not initialized.');
+  }
+}
+
+export class InvalidInvitationError extends DBError {
+  constructor() {
+    super('INVALID_INVITATION', 'Invitation is invalid.');
+  }
+}
+
+export class InvalidStorageVersionError extends DBError {
+  constructor(expected: number, actual: number) {
+    super('INVALID_STORAGE_VERSION', 'Invalid storage version.', { expected, actual });
+  }
+}
+
+export class SpaceNotFoundError extends DBError {
+  constructor(spaceKey: PublicKey) {
+    super('SPACE_NOT_FOUND', 'Space not found.', { spaceKey });
+  }
+}
+
+export class EntityNotFoundError extends DBError {
+  constructor(entityId: ItemID) {
+    super('ITEM_NOT_FOUND', 'Item not found.', { entityId });
+  }
+}
+
+export class UnknownModelError extends DBError {
+  constructor(model: string) {
+    super('UNKNOWN_MODEL', 'Unknown model.', { model });
+  }
+}

package/src/errors/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './errors';

package/src/index.ts

@@ -0,0 +1,10 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+export * from './errors';
+export * from './metadata';
+export * from './pipeline';
+export * from './space';
+export * from './common';
+export * from './dbhost';

package/src/metadata/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export * from './metadata-store';