@dxos/echo-pipeline 0.1.24-next.6b1f434

package/src/metadata/metadata-store.ts

@@ -0,0 +1,173 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { synchronized } from '@dxos/async';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { schema } from '@dxos/protocols';
+import { EchoMetadata, SpaceMetadata } from '@dxos/protocols/proto/dxos/echo/metadata';
+import { IdentityRecord } from '@dxos/protocols/proto/dxos/halo/credentials';
+import { Directory } from '@dxos/random-access-storage';
+import { Timeframe } from '@dxos/timeframe';
+
+/**
+ * Version for the schema of the stored data as defined in dxos.echo.metadata.EchoMetadata.
+ *
+ * Should be incremented every time there's a breaking change to the stored data.
+ */
+export const STORAGE_VERSION = 1;
+
+export interface AddSpaceOptions {
+  key: PublicKey;
+  genesisFeed: PublicKey;
+}
+
+const emptyEchoMetadata = (): EchoMetadata => ({
+  version: STORAGE_VERSION,
+  spaces: [],
+  created: new Date(),
+  updated: new Date()
+});
+
+export class MetadataStore {
+  private _metadata: EchoMetadata = emptyEchoMetadata();
+
+  // prettier-ignore
+  constructor(
+    private readonly _directory: Directory
+  ) {}
+
+  get version(): number {
+    return this._metadata.version ?? 0;
+  }
+
+  /**
+   * Returns a list of currently saved spaces. The list and objects in it can be modified addSpace and
+   * addSpaceFeed functions.
+   */
+  get spaces(): SpaceMetadata[] {
+    return this._metadata.spaces ?? [];
+  }
+
+  /**
+   * Loads metadata from persistent storage.
+   */
+  @synchronized
+  async load(): Promise<void> {
+    const file = this._directory.getOrCreateFile('EchoMetadata');
+    try {
+      const { size: fileLength } = await file.stat();
+      if (fileLength < 4) {
+        return;
+      }
+      // Loading file size from first 4 bytes.
+      const dataSize = fromBytesInt32(await file.read(0, 4));
+      log('loaded', { size: dataSize });
+
+      // Sanity check.
+      {
+        if (fileLength < dataSize + 4) {
+          throw new Error('Metadata storage is corrupted');
+        }
+      }
+
+      const data = await file.read(4, dataSize);
+      this._metadata = schema.getCodecForType('dxos.echo.metadata.EchoMetadata').decode(data);
+    } catch (err: any) {
+      log.error('failed to load metadata', { err });
+      this._metadata = emptyEchoMetadata();
+    } finally {
+      await file.close();
+    }
+  }
+
+  @synchronized
+  private async _save(): Promise<void> {
+    const data: EchoMetadata = {
+      ...this._metadata,
+      version: STORAGE_VERSION,
+      created: this._metadata.created ?? new Date(),
+      updated: new Date()
+    };
+
+    const file = this._directory.getOrCreateFile('EchoMetadata');
+
+    try {
+      const encoded = Buffer.from(schema.getCodecForType('dxos.echo.metadata.EchoMetadata').encode(data));
+
+      // Saving file size at first 4 bytes.
+      await file.write(0, toBytesInt32(encoded.length));
+      log('saved', { size: encoded.length });
+
+      // Saving data.
+      await file.write(4, encoded);
+    } finally {
+      await file.close();
+    }
+  }
+
+  /**
+   * Clears storage - doesn't work for now.
+   */
+  async clear(): Promise<void> {
+    log('clearing all metadata');
+    await this._directory.delete();
+  }
+
+  getIdentityRecord(): IdentityRecord | undefined {
+    return this._metadata.identity;
+  }
+
+  async setIdentityRecord(record: IdentityRecord) {
+    assert(!this._metadata.identity, 'Cannot overwrite existing identity in metadata');
+
+    this._metadata.identity = record;
+    await this._save();
+  }
+
+  async addSpace(record: SpaceMetadata) {
+    assert(
+      !(this._metadata.spaces ?? []).find((space) => space.key === record.key),
+      'Cannot overwrite existing space in metadata'
+    );
+
+    (this._metadata.spaces ??= []).push(record);
+    await this._save();
+  }
+
+  async setSpaceLatestTimeframe(spaceKey: PublicKey, timeframe: Timeframe) {
+    const space = this.spaces.find((space) => space.key === spaceKey);
+    assert(space, 'Space not found');
+
+    space.dataTimeframe = timeframe;
+    await this._save();
+  }
+
+  async setSpaceSnapshot(spaceKey: PublicKey, snapshot: string) {
+    const space = this.spaces.find((space) => space.key === spaceKey);
+    assert(space, 'Space not found');
+
+    space.snapshot = snapshot;
+    await this._save();
+  }
+
+  async setWritableFeedKeys(spaceKey: PublicKey, controlFeedKey: PublicKey, dataFeedKey: PublicKey) {
+    const space = this.spaces.find((space) => space.key === spaceKey);
+    assert(space, 'Space not found');
+
+    space.controlFeedKey = controlFeedKey;
+    space.dataFeedKey = dataFeedKey;
+    await this._save();
+  }
+}
+
+const toBytesInt32 = (num: number) => {
+  const buf = Buffer.alloc(4);
+  buf.writeInt32LE(num, 0);
+  return buf;
+};
+
+const fromBytesInt32 = (buf: Buffer) => buf.readInt32LE(0);

package/src/pipeline/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+export { Pipeline, PipelineAccessor } from './pipeline';
+export * from './timeframe-clock';

package/src/pipeline/message-selector.ts

@@ -0,0 +1,38 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import debug from 'debug';
+import assert from 'node:assert';
+
+import { FeedBlock, FeedBlockSelector } from '@dxos/feed-store';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+
+import { TimeframeClock } from './timeframe-clock';
+
+const log = debug('dxos:echo-db:message-selector');
+
+/**
+ * The MessageSelector makes sure that we read in a trusted order.
+ * The first message we wish to process is the SpaceGenesis, which will admit a Feed.
+ * As we encounter and process FeedAdmit messages those are added to the Space's trust,
+ * and we begin processing messages from them as well.
+ */
+export const createMessageSelector = (timeframeClock: TimeframeClock): FeedBlockSelector<FeedMessage> => {
+  return (messages: FeedBlock<FeedMessage>[]) => {
+    // Pick the first candidate with a valid timeframe that has no gaps.
+    for (let i = 0; i < messages.length; i++) {
+      const {
+        data: { timeframe }
+      } = messages[i];
+      assert(timeframe);
+
+      if (!timeframeClock.hasGaps(timeframe)) {
+        return i;
+      }
+    }
+
+    // Not ready for this message yet.
+    log('Skipping...');
+  };
+};

package/src/pipeline/pipeline.test.ts

@@ -0,0 +1,71 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { checkType } from '@dxos/debug';
+import { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { describe, test, afterTest } from '@dxos/test';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { TestFeedBuilder } from '../testing';
+import { Pipeline } from './pipeline';
+
+describe('pipeline/Pipeline', () => {
+  test('asynchronous reader & writer without ordering', async () => {
+    const pipeline = new Pipeline(new Timeframe());
+    afterTest(() => pipeline.stop());
+
+    const builder = new TestFeedBuilder();
+    const feedStore = builder.createFeedStore();
+
+    // Remote feeds from other peers.
+    const numFeeds = 5;
+    const messagesPerFeed = 10;
+    for (const feedIdx in range(numFeeds)) {
+      const key = await builder.keyring.createKey();
+      const feed = await feedStore.openFeed(key, { writable: true });
+      void pipeline.addFeed(feed);
+
+      setTimeout(async () => {
+        for (const msgIdx in range(messagesPerFeed)) {
+          await feed.append(
+            checkType<FeedMessage>({
+              timeframe: new Timeframe(),
+              payload: {
+                data: {
+                  object: {
+                    objectId: `${feedIdx}-${msgIdx}`
+                  }
+                }
+              }
+            })
+          );
+        }
+      });
+    }
+
+    // Local feed.
+    const key = await builder.keyring.createKey();
+    const feed = await feedStore.openFeed(key, { writable: true });
+    void pipeline.addFeed(feed);
+    pipeline.setWriteFeed(feed);
+
+    for (const msgIdx in range(messagesPerFeed)) {
+      await pipeline.writer!.write({
+        data: {
+          object: {
+            objectId: `local-${msgIdx}`
+          }
+        }
+      });
+    }
+
+    let msgCount = 0;
+    for await (const _ of pipeline.consume()) {
+      if (++msgCount === numFeeds * messagesPerFeed) {
+        void pipeline.stop();
+      }
+    }
+  });
+});

package/src/pipeline/pipeline.ts

@@ -0,0 +1,249 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { sleep, Trigger } from '@dxos/async';
+import { FeedSetIterator, FeedWrapper, FeedWriter } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FeedMessageBlock } from '@dxos/protocols';
+import type { FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { Timeframe } from '@dxos/timeframe';
+
+import { createMappedFeedWriter } from '../common';
+import { createMessageSelector } from './message-selector';
+import { mapFeedIndexesToTimeframe, mapTimeframeToFeedIndexes, TimeframeClock } from './timeframe-clock';
+
+export type WaitUntilReachedTargetParams = {
+  timeout?: number;
+};
+
+/**
+ * External state accessor.
+ */
+export class PipelineState {
+  // TODO(dmaretskyi): Remove?.
+  public readonly timeframeUpdate = this._timeframeClock.update;
+
+  /**
+   * Target timeframe we are waiting to reach.
+   */
+  private _targetTimeframe: Timeframe | undefined;
+
+  private _reachedTargetPromise: Promise<void> | undefined;
+
+  // prettier-ignore
+  constructor(
+    private _iterator: FeedSetIterator<any>,
+    private _timeframeClock: TimeframeClock
+  ) { }
+
+  /**
+   * Latest theoretical timeframe based on the last mutation in each feed.
+   * NOTE: This might never be reached if the mutation dependencies
+   */
+  get endTimeframe() {
+    return mapFeedIndexesToTimeframe(
+      this._iterator.feeds
+        .filter((feed) => feed.properties.length > 0)
+        .map((feed) => ({
+          feedKey: feed.key,
+          index: feed.properties.length - 1
+        }))
+    );
+  }
+
+  get timeframe() {
+    return this._timeframeClock.timeframe;
+  }
+
+  get targetTimeframe() {
+    return this._targetTimeframe ? Timeframe.merge(this.endTimeframe, this._targetTimeframe) : this.endTimeframe;
+  }
+
+  async waitUntilTimeframe(target: Timeframe) {
+    await this._timeframeClock.waitUntilReached(target);
+  }
+
+  setTargetTimeframe(target: Timeframe) {
+    this._targetTimeframe = target;
+  }
+
+  /**
+   * Wait until the pipeline processes all messages in the feed and reaches the target timeframe if that is set.
+   *
+   * This function will resolve immediately if the pipeline is stalled.
+   *
+   * @param timeout Timeout in milliseconds to specify the maximum wait time.
+   */
+  async waitUntilReachedTargetTimeframe({ timeout }: WaitUntilReachedTargetParams = {}) {
+    log('waitUntilReachedTargetTimeframe', {
+      timeout,
+      current: this.timeframe,
+      target: this.targetTimeframe
+    });
+
+    this._reachedTargetPromise ??= Promise.race([
+      this._timeframeClock.update.waitForCondition(() => {
+        return Timeframe.dependencies(this.targetTimeframe, this.timeframe).isEmpty();
+      }),
+      this._iterator.stalled.discardParameter().waitForCount(1)
+    ]);
+
+    let done = false;
+
+    if (timeout) {
+      return Promise.race([
+        this._reachedTargetPromise.then(() => {
+          done = true;
+        }),
+        sleep(timeout).then(() => {
+          if (done) {
+            return;
+          }
+
+          log.warn('waitUntilReachedTargetTimeframe timed out', {
+            timeout,
+            current: this.timeframe,
+            target: this.targetTimeframe,
+            dependencies: Timeframe.dependencies(this.targetTimeframe, this.timeframe)
+          });
+        })
+      ]);
+    } else {
+      return this._reachedTargetPromise;
+    }
+  }
+}
+
+export interface PipelineAccessor {
+  state: PipelineState;
+  writer: FeedWriter<FeedMessage.Payload>;
+}
+
+/**
+ * A multi-reader pipeline that operates on feeds.
+ * Might have a single writable feed.
+ * Has a timeframe clock to handle message ordering.
+ *
+ * NOTE:
+ *  - Feeds passed in must have value encoding consistent with the type expected by the iterator/writer.
+ *
+ * # Usage examples
+ *
+ * ## Create a new space.
+ *
+ * 1. Generate space key, genesis feed key.
+ * 2. Create and open pipeline reading from {}.
+ * 3. Create and add the writable genesis feed.
+ * 4. Write the initial sequence of control and credential messages.
+ *
+ * ## Load an existing space from storage
+ *
+ * 1. Load space key, genesis feed key, get starting timeframe from saved snapshot.
+ * 2. Create and open pipeline reading from the initial timeframe.
+ * 3. Open and add the genesis feed.
+ * 4. Once the writable feed is added, the pipeline becomes writable.
+ *
+ * ## Join an existing space created by another agent/device.
+ *
+ * 1. Get the space key, genesis feed key from another agent.
+ * 2. (optionally) Download the snapshot from another agent.
+ * 3. Create and open pipeline.
+ * 4. Generate the writable feed key.
+ * 5. Wait for the writable feed to be added.
+ */
+export class Pipeline implements PipelineAccessor {
+  private readonly _timeframeClock = new TimeframeClock(this._initialTimeframe);
+
+  // Inbound feed stream.
+  private readonly _feedSetIterator = new FeedSetIterator<FeedMessage>(createMessageSelector(this._timeframeClock), {
+    start: mapTimeframeToFeedIndexes(this._initialTimeframe),
+    stallTimeout: 1000
+  });
+
+  // External state accessor.
+  private readonly _state: PipelineState = new PipelineState(this._feedSetIterator, this._timeframeClock);
+
+  // Waits for the message consumer to process the message and yield control back to the pipeline.
+  private readonly _processingTrigger = new Trigger().wake();
+
+  // Outbound feed writer.
+  private _writer: FeedWriter<FeedMessage.Payload> | undefined;
+
+  private _isOpen = false;
+
+  // prettier-ignore
+  constructor(
+    private readonly _initialTimeframe: Timeframe = new Timeframe()
+  ) {
+    this._feedSetIterator.stalled.on((iterator) => {
+      log.warn(`Stalled after ${iterator.options.stallTimeout}ms with ${iterator.size} feeds.`);
+    });
+  }
+
+  get state() {
+    return this._state;
+  }
+
+  get writer(): FeedWriter<FeedMessage.Payload> {
+    assert(this._writer, 'Writer not set.');
+    return this._writer;
+  }
+
+  getFeeds(): FeedWrapper<FeedMessage>[] {
+    return this._feedSetIterator.feeds;
+  }
+
+  async addFeed(feed: FeedWrapper<FeedMessage>) {
+    await this._feedSetIterator.addFeed(feed);
+  }
+
+  setWriteFeed(feed: FeedWrapper<FeedMessage>) {
+    assert(!this._writer, 'Writer already set.');
+    assert(feed.properties.writable, 'Feed must be writable.');
+
+    this._writer = createMappedFeedWriter<FeedMessage.Payload, FeedMessage>(
+      (payload: FeedMessage.Payload) => ({
+        timeframe: this._timeframeClock.timeframe,
+        payload
+      }),
+      feed.createFeedWriter()
+    );
+  }
+
+  async start() {
+    log('starting...', {});
+    await this._feedSetIterator.open();
+    log('started');
+  }
+
+  async stop() {
+    log('stopping...', {});
+    await this._feedSetIterator.close();
+    await this._processingTrigger.wait(); // Wait for the in-flight message to be processed.
+    log('stopped');
+  }
+
+  /**
+   * Starts to iterate over the ordered messages from the added feeds.
+   * Updates the timeframe clock after the message has bee processed.
+   */
+  async *consume(): AsyncIterable<FeedMessageBlock> {
+    assert(!this._isOpen, 'Pipeline is already being consumed.');
+    this._isOpen = true;
+
+    for await (const block of this._feedSetIterator) {
+      this._processingTrigger.reset();
+      yield block;
+      this._processingTrigger.wake();
+
+      this._timeframeClock.updateTimeframe(PublicKey.from(block.feedKey), block.seq);
+    }
+
+    // TODO(burdon): Test re-entrant?
+    this._isOpen = false;
+  }
+}

package/src/pipeline/timeframe-clock.ts

@@ -0,0 +1,56 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { timed } from '@dxos/debug';
+import { FeedIndex } from '@dxos/feed-store';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Timeframe } from '@dxos/timeframe';
+
+export const mapTimeframeToFeedIndexes = (timeframe: Timeframe): FeedIndex[] =>
+  timeframe.frames().map(([feedKey, index]) => ({ feedKey, index }));
+
+export const mapFeedIndexesToTimeframe = (indexes: FeedIndex[]): Timeframe =>
+  new Timeframe(indexes.map(({ feedKey, index }) => [feedKey, index]));
+
+/**
+ * Keeps state of the last timeframe that was processed by ECHO.
+ */
+export class TimeframeClock {
+  readonly update = new Event<Timeframe>();
+
+  // prettier-ignore
+  constructor(
+    private _timeframe = new Timeframe()
+  ) {}
+
+  get timeframe() {
+    return this._timeframe;
+  }
+
+  updateTimeframe(key: PublicKey, seq: number) {
+    this._timeframe = Timeframe.merge(this._timeframe, new Timeframe([[key, seq]]));
+    this.update.emit(this._timeframe);
+  }
+
+  hasGaps(timeframe: Timeframe) {
+    const gaps = Timeframe.dependencies(timeframe, this._timeframe);
+    return !gaps.isEmpty();
+  }
+
+  @timed(5_000)
+  async waitUntilReached(target: Timeframe) {
+    log.debug('waitUntilReached', { target, current: this._timeframe });
+    await this.update.waitForCondition(() => {
+      log('check if reached', {
+        target,
+        current: this._timeframe,
+        deps: Timeframe.dependencies(target, this._timeframe)
+      });
+
+      return Timeframe.dependencies(target, this._timeframe).isEmpty();
+    });
+  }
+}

package/src/space/auth.ts

@@ -0,0 +1,88 @@
+//
+// Copyright 2019 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { runInContext, scheduleTask } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { randomBytes } from '@dxos/crypto';
+import { log } from '@dxos/log';
+import { schema } from '@dxos/protocols';
+import { AuthService } from '@dxos/protocols/proto/dxos/mesh/teleport/auth';
+import { ExtensionContext, RpcExtension } from '@dxos/teleport';
+
+export type AuthProvider = (nonce: Uint8Array) => Promise<Uint8Array | undefined>;
+
+export type AuthVerifier = (nonce: Uint8Array, credential: Uint8Array) => Promise<boolean>;
+
+export type AuthExtensionParams = {
+  provider: AuthProvider;
+  verifier: AuthVerifier;
+  onAuthSuccess: () => void;
+  onAuthFailure: () => void;
+};
+
+export class AuthExtension extends RpcExtension<Services, Services> {
+  private readonly _ctx = new Context({
+    onError: (err) => {
+      log.catch(err);
+    }
+  });
+
+  constructor(private readonly _authParams: AuthExtensionParams) {
+    super({
+      requested: {
+        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService')
+      },
+      exposed: {
+        AuthService: schema.getService('dxos.mesh.teleport.auth.AuthService')
+      },
+      timeout: 60 * 1000 // Long timeout because auth can wait for sync in certain cases.
+    });
+  }
+
+  protected async getHandlers(): Promise<Services> {
+    return {
+      AuthService: {
+        authenticate: async ({ challenge }) => {
+          try {
+            const credential = await this._authParams.provider(challenge);
+            if (!credential) {
+              throw new Error('auth rejected');
+            }
+            return { credential };
+          } catch (err) {
+            log.error('failed to generate auth credentials', err);
+            throw new Error('auth rejected');
+          }
+        }
+      }
+    };
+  }
+
+  override async onOpen(context: ExtensionContext): Promise<void> {
+    await super.onOpen(context);
+    scheduleTask(this._ctx, async () => {
+      try {
+        const challenge = randomBytes(32);
+        const { credential } = await this.rpc.AuthService.authenticate({ challenge });
+        assert(credential?.length > 0, 'invalid credential');
+        const success = await this._authParams.verifier(challenge, credential);
+        assert(success, 'credential not verified');
+        runInContext(this._ctx, () => this._authParams.onAuthSuccess());
+      } catch (err) {
+        log.warn('auth failed', err);
+        this.close();
+        this._authParams.onAuthFailure();
+      }
+    });
+  }
+
+  override async onClose(): Promise<void> {
+    await this._ctx.dispose();
+    await super.onClose();
+  }
+}
+
+type Services = { AuthService: AuthService };