@dxos/client-services 0.6.13 → 0.6.14-main.69511f5

package/src/packlets/spaces/edge-feed-replicator.test.ts

@@ -0,0 +1,252 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+import { decode as decodeCbor, encode as encodeCbor } from 'cbor-x';
+import { getPort } from 'get-port-please';
+import { describe, expect, onTestFinished, test, vi } from 'vitest';
+
+import { sleep, Trigger } from '@dxos/async';
+import { Context } from '@dxos/context';
+import { valueEncoding } from '@dxos/echo-pipeline';
+import { createEphemeralEdgeIdentity, EdgeClient, EdgeIdentityChangedError } from '@dxos/edge-client';
+import { createTestEdgeWsServer } from '@dxos/edge-client/testing';
+import { FeedFactory, FeedStore, type FeedWrapper } from '@dxos/feed-store';
+import { Keyring } from '@dxos/keyring';
+import { SpaceId } from '@dxos/keys';
+import { type FeedMessage } from '@dxos/protocols/proto/dxos/echo/feed';
+import { createStorage } from '@dxos/random-access-storage';
+import { openAndClose } from '@dxos/test-utils';
+import { Timeframe } from '@dxos/timeframe';
+import { range } from '@dxos/util';
+
+import { EdgeFeedReplicator } from './edge-feed-replicator';
+
+describe('EdgeFeedReplicator', () => {
+  test('requests metadata after connection is open', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    await sleep(50);
+
+    expect(sendSpy).not.toHaveBeenCalled();
+    expect(messageSink.length).toEqual(0);
+
+    admitConnection.wake();
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(1);
+    expect(messageSink.length).toEqual(1);
+    expect(messageSink[0].type).toEqual('get-metadata');
+  });
+
+  test('replicates if added to a connected client', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+    admitConnection.wake();
+    await expect.poll(() => messenger.isConnected).toBeTruthy();
+
+    await attachReplicator(messenger);
+    await expect.poll(() => messageSink.length).toEqual(1);
+  });
+
+  test('sends a block', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+
+    admitConnection.wake();
+    await appendMessage(feed);
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink[1].type).toEqual('data');
+  });
+
+  test('re-requests metadata on reconnect', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    admitConnection.wake();
+    await expect.poll(() => messageSink.length).toEqual(1);
+
+    await updateIdentity(messenger);
+    await messenger.reconnect.waitForCount(1);
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink[1].type).toEqual('get-metadata');
+  });
+
+  test('recovers after query sending failure during identity change', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+
+    sendSpy.mockImplementationOnce(() => {
+      throw new EdgeIdentityChangedError(); // Hard to mock the exact race condition for when this error is thrown
+    });
+    admitConnection.wake();
+
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(1);
+    expect(messageSink.length).toEqual(0);
+    await updateIdentity(messenger);
+
+    await expect.poll(() => messageSink.length).toEqual(1);
+    expect(messageSink[0].type).toEqual('get-metadata');
+  });
+
+  test('recovers after response sending failure during identity change', async () => {
+    const { endpoint, admitConnection, messageSink, sendResponseMessage } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    await appendMessage(feed);
+
+    sendSpy.mockImplementationOnce(async (request: any) => {
+      sendResponseMessage(request, encodeCbor({ type: 'metadata', feedKey: feed.key.toHex(), length: 0 }));
+      return Promise.resolve();
+    });
+    sendSpy.mockImplementationOnce(async () => {
+      throw new EdgeIdentityChangedError();
+    });
+    admitConnection.wake();
+
+    await expect.poll(() => sendSpy.mock.calls.length).toEqual(2);
+    sendSpy.mockRestore();
+    expect(messageSink.length).toEqual(0);
+    await updateIdentity(messenger);
+
+    await messenger.reconnect.waitForCount(1);
+    await expect.poll(() => messageSink.find((msg) => msg.type === 'data')).toBeDefined();
+  });
+
+  test('propagates errors unrelated to reconnect', async () => {
+    const { endpoint, admitConnection } = await createEdge();
+    const { messenger, sendSpy } = await createClient(endpoint);
+
+    const { replicator } = await attachReplicator(messenger, { skipOpen: true });
+    const raised = new Trigger();
+    await replicator.open(new Context({ onError: () => raised.wake() }));
+    onTestFinished(async () => {
+      await replicator.close();
+    });
+
+    sendSpy.mockImplementationOnce(() => {
+      throw new Error();
+    });
+    admitConnection.wake();
+
+    await raised.wait();
+  });
+
+  test('identity update before connected', async () => {
+    const { endpoint, admitConnection, messageSink } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    await attachReplicator(messenger);
+    await updateIdentity(messenger);
+    await sleep(100);
+    admitConnection.wake();
+
+    await expect.poll(() => messageSink.length).toEqual(2);
+    expect(messageSink.map((m) => m.type)).toStrictEqual(range(2, () => 'get-metadata'));
+  });
+
+  test('block appended during reconnect', async () => {
+    const { endpoint, admitConnection, feedLength } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    admitConnection.wake();
+    await sleep(10);
+
+    admitConnection.reset();
+    await updateIdentity(messenger);
+    await appendMessage(feed);
+    await sleep(20);
+    admitConnection.wake();
+
+    await expect.poll(() => feedLength()).toEqual(1);
+  });
+
+  test('reconnect during block append', async () => {
+    const { endpoint, admitConnection, feedLength } = await createEdge();
+    const { messenger } = await createClient(endpoint);
+
+    const { feed } = await attachReplicator(messenger);
+    admitConnection.wake();
+    await sleep(10);
+
+    void appendMessage(feed);
+    await updateIdentity(messenger);
+
+    await expect.poll(() => feedLength()).toEqual(1);
+  });
+
+  const createEdge = async () => {
+    const port = await getPort({ host: 'localhost', port: 7200, portRange: [7200, 7299] });
+    let lastBlockIndex = -1;
+    const admitConnection = new Trigger();
+    const { cleanup, endpoint, messageSink, sendResponseMessage } = await createTestEdgeWsServer(port, {
+      admitConnection,
+      payloadDecoder: decodeCbor,
+      messageHandler: async (message: any) => {
+        if (message.type === 'get-metadata') {
+          return encodeCbor({ type: 'metadata', feedKey: message.feedKey, length: lastBlockIndex + 1 });
+        } else {
+          lastBlockIndex = Math.max(lastBlockIndex, message.blocks[message.blocks.length - 1].index);
+        }
+      },
+    });
+    onTestFinished(cleanup);
+
+    return {
+      endpoint,
+      messageSink,
+      admitConnection,
+      sendResponseMessage,
+      feedLength: () => lastBlockIndex + 1,
+    };
+  };
+
+  const createClient = async (endpoint: string) => {
+    const messenger = new EdgeClient(await createEphemeralEdgeIdentity(), { socketEndpoint: endpoint });
+    const sendSpy = vi.spyOn(messenger, 'send');
+    await openAndClose(messenger);
+    return { messenger, sendSpy };
+  };
+
+  const attachReplicator = async (messenger: EdgeClient, options?: { skipOpen?: boolean }) => {
+    const spaceId = SpaceId.random();
+    const feed = await createNewFeed();
+    const replicator = new EdgeFeedReplicator({ messenger, spaceId });
+    await replicator.addFeed(feed);
+    if (!options?.skipOpen) {
+      await openAndClose(replicator);
+    }
+    return { feed, replicator };
+  };
+
+  const createNewFeed = async () => {
+    const storage = createStorage();
+    const keyring = new Keyring();
+    const feedStore = new FeedStore<FeedMessage>({
+      factory: new FeedFactory<FeedMessage>({
+        root: storage.createDirectory(),
+        signer: keyring,
+        hypercore: { valueEncoding },
+      }),
+    });
+    onTestFinished(() => feedStore.close());
+    return feedStore.openFeed(await keyring.createKey(), { writable: true });
+  };
+
+  const updateIdentity = async (messenger: EdgeClient) => {
+    messenger.setIdentity(await createEphemeralEdgeIdentity());
+  };
+
+  const appendMessage = (feed: FeedWrapper<FeedMessage>) => feed.append({ timeframe: new Timeframe() });
+});

package/src/packlets/spaces/edge-feed-replicator.ts

@@ -5,15 +5,19 @@
 import { decode as decodeCbor, encode as encodeCbor } from 'cbor-x';
 
 import { Event, Mutex, scheduleMicroTask } from '@dxos/async';
-import { Resource, type Context } from '@dxos/context';
+import { Context, Resource } from '@dxos/context';
 import { type EdgeConnection } from '@dxos/edge-client';
+import { EdgeConnectionClosedError, EdgeIdentityChangedError } from '@dxos/edge-client';
 import { type FeedWrapper } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { PublicKey, type SpaceId } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { log, logInfo } from '@dxos/log';
 import { EdgeService } from '@dxos/protocols';
 import { buf } from '@dxos/protocols/buf';
-import { MessageSchema as RouterMessageSchema } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+import {
+  MessageSchema as RouterMessageSchema,
+  type Message as RouterMessage,
+} from '@dxos/protocols/buf/dxos/edge/messenger_pb';
 import type { FeedBlock, ProtocolMessage } from '@dxos/protocols/feed-replication';
 import { ComplexMap, arrayToBuffer, bufferToArray, defaultMap, rangeFromTo } from '@dxos/util';
 
@@ -24,7 +28,10 @@ export type EdgeFeedReplicatorParams = {
 
 export class EdgeFeedReplicator extends Resource {
   private readonly _messenger: EdgeConnection;
+
+  @logInfo
   private readonly _spaceId: SpaceId;
+
   private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);
 
   private _connectionCtx?: Context = undefined;
@@ -46,9 +53,10 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   protected override async _open(): Promise<void> {
+    log('open');
     // TODO: handle reconnects
     this._ctx.onDispose(
-      this._messenger.addListener(async (message) => {
+      this._messenger.addListener((message: RouterMessage) => {
         if (!message.serviceId) {
           return;
         }
@@ -64,21 +72,40 @@ export class EdgeFeedReplicator extends Resource {
         }
 
         const payload = decodeCbor(message.payload!.value) as ProtocolMessage;
-        log.info('recv', { from: message.source, payload });
+        log('receive', { from: message.source, feedKey: payload.feedKey, type: payload.type });
         this._onMessage(payload);
       }),
     );
 
-    this._connected = true;
-    this._connectionCtx = this._ctx.derive();
-    for (const feed of this._feeds.values()) {
-      await this._replicateFeed(feed);
+    this._messenger.connected.on(this._ctx, async () => {
+      await this._resetConnection();
+      this._startReplication();
+    });
+
+    if (this._messenger.isConnected) {
+      this._startReplication();
     }
   }
 
   protected override async _close(): Promise<void> {
-    this._connected = false;
+    log('close');
+    await this._resetConnection();
+  }
 
+  private _startReplication() {
+    this._connected = true;
+    const connectionCtx = this._createConnectionContext();
+    this._connectionCtx = connectionCtx;
+    log('connection context created');
+    scheduleMicroTask(connectionCtx, async () => {
+      for (const feed of this._feeds.values()) {
+        await this._replicateFeed(connectionCtx, feed);
+      }
+    });
+  }
+
+  private async _resetConnection() {
+    log('resetConnection');
     this._connected = false;
     await this._connectionCtx?.dispose();
     this._connectionCtx = undefined;
@@ -86,11 +113,11 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   async addFeed(feed: FeedWrapper<any>) {
-    log.info('addFeed', { key: feed.key });
+    log.info('addFeed', { key: feed.key, connected: this._connected, hasConnectionCtx: !!this._connectionCtx });
     this._feeds.set(feed.key, feed);
 
-    if (this._connected) {
-      await this._replicateFeed(feed);
+    if (this._connected && this._connectionCtx) {
+      await this._replicateFeed(this._connectionCtx, feed);
     }
   }
 
@@ -98,25 +125,32 @@ export class EdgeFeedReplicator extends Resource {
     return defaultMap(this._pushMutex, key, () => new Mutex());
   }
 
-  private async _replicateFeed(feed: FeedWrapper<any>) {
-    invariant(this._connectionCtx);
-
+  private async _replicateFeed(ctx: Context, feed: FeedWrapper<any>) {
+    log('replicateFeed', { key: feed.key });
     await this._sendMessage({
       type: 'get-metadata',
       feedKey: feed.key.toHex(),
     });
 
-    Event.wrap(feed.core as any, 'append').on(this._connectionCtx, async () => {
+    Event.wrap(feed.core as any, 'append').on(ctx, async () => {
       await this._pushBlocksIfNeeded(feed);
     });
   }
 
   private async _sendMessage(message: ProtocolMessage) {
-    log.info('sending message', { message });
+    if (!this._connectionCtx) {
+      log.info('message dropped because connection was disposed');
+      return;
+    }
+
+    const logPayload =
+      message.type === 'data' ? { feedKey: message.feedKey, blocks: message.blocks.map((b) => b.index) } : { message };
+    log.info('sending message', logPayload);
 
     invariant(message.feedKey);
     const payloadValue = bufferToArray(encodeCbor(message));
 
+    log('send', { type: message.type });
     await this._messenger.send(
       buf.create(RouterMessageSchema, {
         source: {
@@ -130,11 +164,15 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   private _onMessage(message: ProtocolMessage) {
-    log.info('received message', { message });
-
-    scheduleMicroTask(this._ctx, async () => {
+    if (!this._connectionCtx) {
+      log.warn('received message after connection context was disposed');
+      return;
+    }
+    scheduleMicroTask(this._connectionCtx, async () => {
       switch (message.type) {
         case 'metadata': {
+          log.info('received metadata', { message });
+
           const feedKey = PublicKey.fromHex(message.feedKey);
           const feed = this._feeds.get(feedKey);
           if (!feed) {
@@ -160,6 +198,8 @@ export class EdgeFeedReplicator extends Resource {
         }
 
         case 'data': {
+          log.info('received data', { feed: message.feedKey, blocks: message.blocks.map((b) => b.index) });
+
           const feedKey = PublicKey.fromHex(message.feedKey);
           const feed = this._feeds.get(feedKey);
           if (!feed) {
@@ -223,9 +263,10 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   private async _pushBlocksIfNeeded(feed: FeedWrapper<any>) {
-    using _guard = await this._getPushMutex(feed.key).acquire();
+    using _ = await this._getPushMutex(feed.key).acquire();
 
     if (!this._remoteLength.has(feed.key)) {
+      log('blocks not pushed because remote length is unknown');
       return;
     }
 
@@ -234,6 +275,23 @@ export class EdgeFeedReplicator extends Resource {
       await this._pushBlocks(feed, remoteLength, feed.length);
     }
   }
+
+  private _createConnectionContext() {
+    const connectionCtx = new Context({
+      onError: async (err: any) => {
+        if (connectionCtx !== this._connectionCtx) {
+          return;
+        }
+        if (err instanceof EdgeIdentityChangedError || err instanceof EdgeConnectionClosedError) {
+          log('resetting on reconnect');
+          await this._resetConnection();
+        } else {
+          this._ctx.raise(err);
+        }
+      },
+    });
+    return connectionCtx;
+  }
 }
 
 // hypercore requires buffers

package/src/packlets/spaces/epoch-migrations.ts

@@ -4,13 +4,13 @@
 
 import type { AutomergeUrl } from '@dxos/automerge/automerge-repo';
 import { type Context } from '@dxos/context';
+import { migrateDocument } from '@dxos/echo-db';
 import {
   convertLegacyReferences,
   convertLegacySpaceRootDoc,
   findInlineObjectOfType,
-  migrateDocument,
   type EchoHost,
-} from '@dxos/echo-db';
+} from '@dxos/echo-pipeline';
 import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
 import { TYPE_PROPERTIES } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';

package/src/packlets/spaces/notarization-plugin.test.ts

@@ -2,27 +2,28 @@
 // Copyright 2023 DXOS.org
 //
 
-import { expect } from 'chai';
+import { onTestFinished, describe, expect, test } from 'vitest';
 
 import { Context } from '@dxos/context';
 import { CredentialGenerator } from '@dxos/credentials';
 import { MockFeedWriter } from '@dxos/feed-store/testing';
 import { Keyring } from '@dxos/keyring';
+import { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { TestBuilder, type TestConnection, TestPeer } from '@dxos/teleport/testing';
-import { afterTest, describe, test } from '@dxos/test';
 
-import { NotarizationPlugin } from './notarization-plugin';
+import { NotarizationPlugin, type NotarizationPluginParams } from './notarization-plugin';
 
 class TestAgent extends TestPeer {
   private readonly _ctx = new Context();
 
   feed = new MockFeedWriter<Credential>();
-  notarizationPlugin = new NotarizationPlugin();
+  notarizationPlugin: NotarizationPlugin;
 
-  constructor() {
+  constructor(params: NotarizationPluginParams) {
     super();
+    this.notarizationPlugin = new NotarizationPlugin(params);
     this.feed.written.on(this._ctx, async ([credential]) => {
       log('written to feed', { credential });
       await this.notarizationPlugin.processCredential(credential);
@@ -49,10 +50,12 @@ class TestAgent extends TestPeer {
 describe('NotarizationPlugin', () => {
   test('notarize single credential', async () => {
     const testBuilder = new TestBuilder();
-    afterTest(() => testBuilder.destroy());
+    onTestFinished(() => testBuilder.destroy());
+
+    const params = { spaceId: SpaceId.random() };
 
     // peer0 is there to test retries.
-    const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent() });
+    const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent(params) });
     peer1.enableWriting();
 
     peer1.feed.written.on(async ([credential]) => {