@dxos/client-services 0.6.13 → 0.6.14-main.1366248

package/src/packlets/spaces/edge-feed-replicator.ts

@@ -5,15 +5,19 @@
 import { decode as decodeCbor, encode as encodeCbor } from 'cbor-x';
 
 import { Event, Mutex, scheduleMicroTask } from '@dxos/async';
-import { Resource, type Context } from '@dxos/context';
+import { Context, Resource } from '@dxos/context';
 import { type EdgeConnection } from '@dxos/edge-client';
+import { EdgeConnectionClosedError, EdgeIdentityChangedError } from '@dxos/edge-client';
 import { type FeedWrapper } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { PublicKey, type SpaceId } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { log, logInfo } from '@dxos/log';
 import { EdgeService } from '@dxos/protocols';
 import { buf } from '@dxos/protocols/buf';
-import { MessageSchema as RouterMessageSchema } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+import {
+  MessageSchema as RouterMessageSchema,
+  type Message as RouterMessage,
+} from '@dxos/protocols/buf/dxos/edge/messenger_pb';
 import type { FeedBlock, ProtocolMessage } from '@dxos/protocols/feed-replication';
 import { ComplexMap, arrayToBuffer, bufferToArray, defaultMap, rangeFromTo } from '@dxos/util';
 
@@ -24,7 +28,10 @@ export type EdgeFeedReplicatorParams = {
 
 export class EdgeFeedReplicator extends Resource {
   private readonly _messenger: EdgeConnection;
+
+  @logInfo
   private readonly _spaceId: SpaceId;
+
   private readonly _feeds = new ComplexMap<PublicKey, FeedWrapper<any>>(PublicKey.hash);
 
   private _connectionCtx?: Context = undefined;
@@ -46,9 +53,10 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   protected override async _open(): Promise<void> {
+    log('open');
     // TODO: handle reconnects
     this._ctx.onDispose(
-      this._messenger.addListener(async (message) => {
+      this._messenger.onMessage((message: RouterMessage) => {
         if (!message.serviceId) {
           return;
         }
@@ -64,21 +72,44 @@ export class EdgeFeedReplicator extends Resource {
         }
 
         const payload = decodeCbor(message.payload!.value) as ProtocolMessage;
-        log.info('recv', { from: message.source, payload });
+        log('receive', { from: message.source, feedKey: payload.feedKey, type: payload.type });
         this._onMessage(payload);
       }),
     );
 
-    this._connected = true;
-    this._connectionCtx = this._ctx.derive();
-    for (const feed of this._feeds.values()) {
-      await this._replicateFeed(feed);
+    this._ctx.onDispose(
+      this._messenger.onReconnected(() => {
+        scheduleMicroTask(this._ctx, () => this._handleReconnect());
+      }),
+    );
+  }
+
+  private async _handleReconnect() {
+    await this._resetConnection();
+    if (this._messenger.isConnected) {
+      this._startReplication();
     }
   }
 
   protected override async _close(): Promise<void> {
-    this._connected = false;
+    log('close');
+    await this._resetConnection();
+  }
+
+  private _startReplication() {
+    this._connected = true;
+    const connectionCtx = this._createConnectionContext();
+    this._connectionCtx = connectionCtx;
+    log('connection context created');
+    scheduleMicroTask(connectionCtx, async () => {
+      for (const feed of this._feeds.values()) {
+        await this._replicateFeed(connectionCtx, feed);
+      }
+    });
+  }
 
+  private async _resetConnection() {
+    log('resetConnection');
     this._connected = false;
     await this._connectionCtx?.dispose();
     this._connectionCtx = undefined;
@@ -86,11 +117,11 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   async addFeed(feed: FeedWrapper<any>) {
-    log.info('addFeed', { key: feed.key });
+    log.info('addFeed', { key: feed.key, connected: this._connected, hasConnectionCtx: !!this._connectionCtx });
     this._feeds.set(feed.key, feed);
 
-    if (this._connected) {
-      await this._replicateFeed(feed);
+    if (this._connected && this._connectionCtx) {
+      await this._replicateFeed(this._connectionCtx, feed);
     }
   }
 
@@ -98,25 +129,32 @@ export class EdgeFeedReplicator extends Resource {
     return defaultMap(this._pushMutex, key, () => new Mutex());
   }
 
-  private async _replicateFeed(feed: FeedWrapper<any>) {
-    invariant(this._connectionCtx);
-
+  private async _replicateFeed(ctx: Context, feed: FeedWrapper<any>) {
+    log('replicateFeed', { key: feed.key });
     await this._sendMessage({
       type: 'get-metadata',
       feedKey: feed.key.toHex(),
     });
 
-    Event.wrap(feed.core as any, 'append').on(this._connectionCtx, async () => {
+    Event.wrap(feed.core as any, 'append').on(ctx, async () => {
       await this._pushBlocksIfNeeded(feed);
     });
   }
 
   private async _sendMessage(message: ProtocolMessage) {
-    log.info('sending message', { message });
+    if (!this._connectionCtx) {
+      log.info('message dropped because connection was disposed');
+      return;
+    }
+
+    const logPayload =
+      message.type === 'data' ? { feedKey: message.feedKey, blocks: message.blocks.map((b) => b.index) } : { message };
+    log.info('sending message', logPayload);
 
     invariant(message.feedKey);
     const payloadValue = bufferToArray(encodeCbor(message));
 
+    log('send', { type: message.type });
     await this._messenger.send(
       buf.create(RouterMessageSchema, {
         source: {
@@ -130,11 +168,15 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   private _onMessage(message: ProtocolMessage) {
-    log.info('received message', { message });
-
-    scheduleMicroTask(this._ctx, async () => {
+    if (!this._connectionCtx) {
+      log.warn('received message after connection context was disposed');
+      return;
+    }
+    scheduleMicroTask(this._connectionCtx, async () => {
       switch (message.type) {
         case 'metadata': {
+          log.info('received metadata', { message });
+
           const feedKey = PublicKey.fromHex(message.feedKey);
           const feed = this._feeds.get(feedKey);
           if (!feed) {
@@ -160,6 +202,8 @@ export class EdgeFeedReplicator extends Resource {
         }
 
         case 'data': {
+          log.info('received data', { feed: message.feedKey, blocks: message.blocks.map((b) => b.index) });
+
           const feedKey = PublicKey.fromHex(message.feedKey);
           const feed = this._feeds.get(feedKey);
           if (!feed) {
@@ -223,9 +267,10 @@ export class EdgeFeedReplicator extends Resource {
   }
 
   private async _pushBlocksIfNeeded(feed: FeedWrapper<any>) {
-    using _guard = await this._getPushMutex(feed.key).acquire();
+    using _ = await this._getPushMutex(feed.key).acquire();
 
     if (!this._remoteLength.has(feed.key)) {
+      log('blocks not pushed because remote length is unknown');
       return;
     }
 
@@ -234,6 +279,23 @@ export class EdgeFeedReplicator extends Resource {
       await this._pushBlocks(feed, remoteLength, feed.length);
     }
   }
+
+  private _createConnectionContext() {
+    const connectionCtx = new Context({
+      onError: async (err: any) => {
+        if (connectionCtx !== this._connectionCtx) {
+          return;
+        }
+        if (err instanceof EdgeIdentityChangedError || err instanceof EdgeConnectionClosedError) {
+          log('resetting on reconnect');
+          await this._resetConnection();
+        } else {
+          this._ctx.raise(err);
+        }
+      },
+    });
+    return connectionCtx;
+  }
 }
 
 // hypercore requires buffers

package/src/packlets/spaces/epoch-migrations.ts

@@ -4,13 +4,13 @@
 
 import type { AutomergeUrl } from '@dxos/automerge/automerge-repo';
 import { type Context } from '@dxos/context';
+import { migrateDocument } from '@dxos/echo-db';
 import {
   convertLegacyReferences,
   convertLegacySpaceRootDoc,
   findInlineObjectOfType,
-  migrateDocument,
   type EchoHost,
-} from '@dxos/echo-db';
+} from '@dxos/echo-pipeline';
 import { SpaceDocVersion, type SpaceDoc } from '@dxos/echo-protocol';
 import { TYPE_PROPERTIES } from '@dxos/echo-schema';
 import { invariant } from '@dxos/invariant';

package/src/packlets/spaces/notarization-plugin.test.ts

@@ -2,27 +2,28 @@
 // Copyright 2023 DXOS.org
 //
 
-import { expect } from 'chai';
+import { onTestFinished, describe, expect, test } from 'vitest';
 
 import { Context } from '@dxos/context';
 import { CredentialGenerator } from '@dxos/credentials';
 import { MockFeedWriter } from '@dxos/feed-store/testing';
 import { Keyring } from '@dxos/keyring';
+import { SpaceId } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { AdmittedFeed, type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { TestBuilder, type TestConnection, TestPeer } from '@dxos/teleport/testing';
-import { afterTest, describe, test } from '@dxos/test';
 
-import { NotarizationPlugin } from './notarization-plugin';
+import { NotarizationPlugin, type NotarizationPluginParams } from './notarization-plugin';
 
 class TestAgent extends TestPeer {
   private readonly _ctx = new Context();
 
   feed = new MockFeedWriter<Credential>();
-  notarizationPlugin = new NotarizationPlugin();
+  notarizationPlugin: NotarizationPlugin;
 
-  constructor() {
+  constructor(params: NotarizationPluginParams) {
     super();
+    this.notarizationPlugin = new NotarizationPlugin(params);
     this.feed.written.on(this._ctx, async ([credential]) => {
       log('written to feed', { credential });
       await this.notarizationPlugin.processCredential(credential);
@@ -49,10 +50,12 @@ class TestAgent extends TestPeer {
 describe('NotarizationPlugin', () => {
   test('notarize single credential', async () => {
     const testBuilder = new TestBuilder();
-    afterTest(() => testBuilder.destroy());
+    onTestFinished(() => testBuilder.destroy());
+
+    const params = { spaceId: SpaceId.random() };
 
     // peer0 is there to test retries.
-    const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent() });
+    const [_peer0, peer1, peer2] = await testBuilder.createPeers({ factory: () => new TestAgent(params) });
     peer1.enableWriting();
 
     peer1.feed.written.on(async ([credential]) => {

package/src/packlets/spaces/notarization-plugin.ts

@@ -2,14 +2,18 @@
 // Copyright 2023 DXOS.org
 //
 
-import { DeferredTask, Event, scheduleTask, sleep, TimeoutError, Trigger } from '@dxos/async';
-import { Context, rejectOnDispose } from '@dxos/context';
-import { type CredentialProcessor } from '@dxos/credentials';
+import { DeferredTask, Event, scheduleTask, sleep, TimeoutError, Trigger, scheduleMicroTask } from '@dxos/async';
+import { type Context, rejectOnDispose, Resource } from '@dxos/context';
+import { type CredentialProcessor, verifyCredential } from '@dxos/credentials';
+import { type EdgeHttpClient } from '@dxos/edge-client';
 import { type FeedWriter } from '@dxos/feed-store';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
+import { type SpaceId } from '@dxos/keys';
+import { logInfo, log } from '@dxos/log';
+import { EdgeCallFailedError } from '@dxos/protocols';
 import { schema } from '@dxos/protocols/proto';
+import { type Runtime } from '@dxos/protocols/proto/dxos/config';
 import { type Credential } from '@dxos/protocols/proto/dxos/halo/credentials';
 import { type NotarizationService, type NotarizeRequest } from '@dxos/protocols/proto/dxos/mesh/teleport/notarization';
 import { type ExtensionContext, RpcExtension } from '@dxos/teleport';
@@ -21,8 +25,21 @@ const DEFAULT_SUCCESS_DELAY = 1_000;
 
 const DEFAULT_NOTARIZE_TIMEOUT = 10_000;
 
+const DEFAULT_ACTIVE_EDGE_POLLING_INTERVAL = 3_000;
+
+const MAX_EDGE_RETRIES = 2;
+
 const WRITER_NOT_SET_ERROR_CODE = 'WRITER_NOT_SET';
 
+const credentialCodec = schema.getCodecForType('dxos.halo.credentials.Credential');
+
+export type NotarizationPluginParams = {
+  spaceId: SpaceId;
+  edgeClient?: EdgeHttpClient;
+  edgeFeatures?: Runtime.Client.EdgeFeatures;
+  activeEdgePollingInterval?: number;
+};
+
 export type NotarizeParams = {
   /**
    * For cancellation.
@@ -53,13 +70,17 @@ export type NotarizeParams = {
    * @default {@link DEFAULT_SUCCESS_DELAY}
    */
   successDelay?: number;
+
+  /**
+   * A random amount of time before making or retrying an edge request to help prevent large bursts of requests.
+   */
+  edgeRetryJitter?: number;
 };
 
 /**
  * See NotarizationService proto.
  */
-export class NotarizationPlugin implements CredentialProcessor {
-  private readonly _ctx = new Context();
+export class NotarizationPlugin extends Resource implements CredentialProcessor {
   private readonly _extensionOpened = new Event();
 
   private _writer: FeedWriter<Credential> | undefined;
@@ -67,13 +88,54 @@ export class NotarizationPlugin implements CredentialProcessor {
   private readonly _processedCredentials = new ComplexSet<PublicKey>(PublicKey.hash);
   private readonly _processCredentialsTriggers = new ComplexMap<PublicKey, Trigger>(PublicKey.hash);
 
+  private _activeEdgePollingIntervalHandle: any | undefined = undefined;
+  private readonly _activeEdgePollingInterval: number;
+
+  @logInfo
+  private readonly _spaceId: SpaceId;
+
+  private readonly _edgeClient: EdgeHttpClient | undefined;
+
+  constructor(params: NotarizationPluginParams) {
+    super();
+    this._spaceId = params.spaceId;
+    this._activeEdgePollingInterval = params.activeEdgePollingInterval ?? DEFAULT_ACTIVE_EDGE_POLLING_INTERVAL;
+    if (params.edgeClient && params.edgeFeatures?.feedReplicator) {
+      this._edgeClient = params.edgeClient;
+    }
+  }
+
+  setActiveEdgePollingEnabled(enabled: boolean) {
+    invariant(this.isOpen);
+    const client = this._edgeClient;
+    invariant(client);
+    if (enabled && !this._activeEdgePollingIntervalHandle) {
+      this._activeEdgePollingIntervalHandle = setInterval(() => {
+        if (this._writer) {
+          this._notarizePendingEdgeCredentials(client, this._writer);
+        }
+      }, this._activeEdgePollingInterval);
+    } else if (!enabled && this._activeEdgePollingIntervalHandle) {
+      clearInterval(this._activeEdgePollingIntervalHandle);
+      this._activeEdgePollingIntervalHandle = undefined;
+    }
+  }
+
   get hasWriter() {
     return !!this._writer;
   }
 
-  async open() {}
+  protected override async _open() {
+    if (this._edgeClient && this._writer) {
+      this._notarizePendingEdgeCredentials(this._edgeClient, this._writer);
+    }
+  }
 
-  async close() {
+  protected override async _close() {
+    if (this._activeEdgePollingIntervalHandle) {
+      clearInterval(this._activeEdgePollingIntervalHandle);
+      this._activeEdgePollingIntervalHandle = undefined;
+    }
     await this._ctx.dispose();
   }
 
@@ -86,6 +148,7 @@ export class NotarizationPlugin implements CredentialProcessor {
     timeout = DEFAULT_NOTARIZE_TIMEOUT,
     retryTimeout = DEFAULT_RETRY_TIMEOUT,
     successDelay = DEFAULT_SUCCESS_DELAY,
+    edgeRetryJitter,
   }: NotarizeParams) {
     log('notarize', { credentials });
     invariant(
@@ -103,24 +166,35 @@ export class NotarizationPlugin implements CredentialProcessor {
     });
     opCtx?.onDispose(() => ctx.dispose());
 
-    // Timeout/
     if (timeout !== 0) {
-      scheduleTask(
-        ctx,
-        () => {
-          log.warn('Notarization timeout', {
-            timeout,
-            peers: Array.from(this._extensions).map((extension) => extension.remotePeerId),
-          });
-          void ctx.dispose();
-          errors.throw(new TimeoutError(timeout, 'Notarization timed out'));
-        },
-        timeout,
-      );
+      this._scheduleTimeout(ctx, errors, timeout);
     }
 
     const allNotarized = Promise.all(credentials.map((credential) => this._waitUntilProcessed(credential.id!)));
 
+    this._tryNotarizeCredentialsWithPeers(ctx, credentials, { retryTimeout, successDelay });
+
+    if (this._edgeClient) {
+      this._tryNotarizeCredentialsWithEdge(ctx, this._edgeClient, credentials, {
+        retryTimeout,
+        successDelay,
+        jitter: edgeRetryJitter,
+      });
+    }
+
+    try {
+      await Promise.race([rejectOnDispose(ctx), allNotarized, errors.wait()]);
+      log('done');
+    } finally {
+      await ctx.dispose();
+    }
+  }
+
+  private _tryNotarizeCredentialsWithPeers(
+    ctx: Context,
+    credentials: Credential[],
+    { retryTimeout, successDelay }: NotarizationTimeouts,
+  ) {
     const peersTried = new Set<NotarizationTeleportExtension>();
 
     // Repeatable task that tries to notarize credentials with one of the available peers.
@@ -145,6 +219,7 @@ export class NotarizationPlugin implements CredentialProcessor {
           credentials: credentials.filter((credential) => !this._processedCredentials.has(credential.id!)),
         });
         log('success');
+
         await sleep(successDelay); // wait before trying with a new peer
       } catch (err: any) {
         if (!ctx.disposed && !err.message.includes(WRITER_NOT_SET_ERROR_CODE)) {
@@ -156,13 +231,31 @@ export class NotarizationPlugin implements CredentialProcessor {
 
     notarizeTask.schedule();
     this._extensionOpened.on(ctx, () => notarizeTask.schedule());
+  }
 
-    try {
-      await Promise.race([rejectOnDispose(ctx), allNotarized, errors.wait()]);
-      log('done');
-    } finally {
-      await ctx.dispose();
-    }
+  private _tryNotarizeCredentialsWithEdge(
+    ctx: Context,
+    client: EdgeHttpClient,
+    credentials: Credential[],
+    timeouts: NotarizationTimeouts & { jitter?: number },
+  ) {
+    const encodedCredentials = credentials.map((credential) => {
+      const binary = credentialCodec.encode(credential);
+      return Buffer.from(binary).toString('base64');
+    });
+    scheduleTask(ctx, async () => {
+      try {
+        await client.notarizeCredentials(
+          this._spaceId,
+          { credentials: encodedCredentials },
+          { retry: { count: MAX_EDGE_RETRIES, timeout: timeouts.retryTimeout, jitter: timeouts.jitter } },
+        );
+
+        log('edge notarization success');
+      } catch (error: any) {
+        handleEdgeError(error);
+      }
+    });
   }
 
   /**
@@ -180,6 +273,44 @@ export class NotarizationPlugin implements CredentialProcessor {
   setWriter(writer: FeedWriter<Credential>) {
     invariant(!this._writer, 'Writer already set.');
     this._writer = writer;
+    if (this._edgeClient) {
+      this._notarizePendingEdgeCredentials(this._edgeClient, writer);
+    }
+  }
+
+  /**
+   * The method is used only for adding agent feeds to spaces.
+   * When an agent is created we can admit them into all the existing spaces. In case the operation fails
+   * this method will fix it on the next space open.
+   * Given how rarely this happens there's no need to poll the endpoint.
+   */
+  private _notarizePendingEdgeCredentials(client: EdgeHttpClient, writer: FeedWriter<Credential>) {
+    scheduleMicroTask(this._ctx, async () => {
+      try {
+        const response = await client.getCredentialsForNotarization(this._spaceId, {
+          retry: { count: MAX_EDGE_RETRIES },
+        });
+
+        const credentials = response.awaitingNotarization.credentials;
+        if (!credentials.length) {
+          log('edge did not return credentials for notarization');
+          return;
+        }
+
+        log('got edge credentials for notarization', { count: credentials.length });
+
+        const decodedCredentials = credentials.map((credential) => {
+          const binary = Buffer.from(credential, 'base64');
+          return credentialCodec.decode(binary);
+        });
+
+        await this._notarizeCredentials(writer, decodedCredentials);
+
+        log.info('notarized edge credentials', { count: decodedCredentials.length });
+      } catch (error: any) {
+        handleEdgeError(error);
+      }
+    });
   }
 
   private async _waitUntilProcessed(id: PublicKey) {
@@ -196,12 +327,20 @@ export class NotarizationPlugin implements CredentialProcessor {
     if (!this._writer) {
       throw new Error(WRITER_NOT_SET_ERROR_CODE);
     }
-    for (const credential of request.credentials ?? []) {
+    await this._notarizeCredentials(this._writer, request.credentials ?? []);
+  }
+
+  private async _notarizeCredentials(writer: FeedWriter<Credential>, credentials: Credential[]) {
+    for (const credential of credentials) {
       invariant(credential.id, 'Credential must have an id');
       if (this._processedCredentials.has(credential.id)) {
         continue;
       }
-      await this._writer.write(credential);
+      const verificationResult = await verifyCredential(credential);
+      if (verificationResult.kind === 'fail') {
+        throw new Error(`Credential verification failed: ${verificationResult.errors.join('\n')}.`);
+      }
+      await writer.write(credential);
     }
   }
 
@@ -220,8 +359,31 @@ export class NotarizationPlugin implements CredentialProcessor {
     });
     return extension;
   }
+
+  private _scheduleTimeout(ctx: Context, errors: Trigger, timeout: number) {
+    scheduleTask(
+      ctx,
+      () => {
+        log.warn('Notarization timeout', {
+          timeout,
+          peers: Array.from(this._extensions).map((extension) => extension.remotePeerId),
+        });
+        void ctx.dispose();
+        errors.throw(new TimeoutError(timeout, 'Notarization timed out'));
+      },
+      timeout,
+    );
+  }
 }
 
+const handleEdgeError = (error: any) => {
+  if (!(error instanceof EdgeCallFailedError) || error.errorData) {
+    log.catch(error);
+  } else {
+    log.info('Edge notarization failure', { reason: error.reason });
+  }
+};
+
 export type NotarizationTeleportExtensionParams = {
   onOpen: () => Promise<void>;
   onClose: () => Promise<void>;
@@ -261,6 +423,11 @@ export class NotarizationTeleportExtension extends RpcExtension<Services, Servic
   }
 }
 
+type NotarizationTimeouts = {
+  retryTimeout: number;
+  successDelay: number;
+};
+
 type Services = {
   NotarizationService: NotarizationService;
 };

package/src/packlets/spaces/spaces-service.test.ts

@@ -2,21 +2,17 @@
 // Copyright 2023 DXOS.org
 //
 
-import chai, { expect } from 'chai';
-import chaiAsPromised from 'chai-as-promised';
+import { afterEach, onTestFinished, beforeEach, describe, expect, test } from 'vitest';
 
 import { Trigger } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { PublicKey } from '@dxos/keys';
 import { type Space, type SpacesService } from '@dxos/protocols/proto/dxos/client/services';
-import { afterEach, afterTest, beforeEach, describe, test } from '@dxos/test';
 
 import { SpacesServiceImpl } from './spaces-service';
 import { type ServiceContext } from '../services';
 import { createServiceContext } from '../testing';
 
-chai.use(chaiAsPromised);
-
 describe('SpacesService', () => {
   let serviceContext: ServiceContext;
   let spacesService: SpacesService;
@@ -36,7 +32,7 @@ describe('SpacesService', () => {
 
   describe('createSpace', () => {
     test('fails if no identity is available', async () => {
-      await expect(spacesService.createSpace()).to.be.rejectedWith();
+      await expect(spacesService.createSpace()).rejects.toBeInstanceOf(Error);
     });
 
     test('creates a new space', async () => {
@@ -56,7 +52,7 @@ describe('SpacesService', () => {
       query.subscribe(({ spaces }) => {
         result.wake(spaces);
       });
-      afterTest(() => query.close());
+      onTestFinished(() => query.close());
       expect(await result.wait()).to.be.length(0);
     });
 
@@ -73,7 +69,7 @@ describe('SpacesService', () => {
       query.subscribe(({ spaces }) => {
         result.wake(spaces);
       });
-      afterTest(() => query.close());
+      onTestFinished(() => query.close());
 
       const spaces = await result.wait();
       expect(spaces).to.be.length(3);
@@ -87,7 +83,7 @@ describe('SpacesService', () => {
       query.subscribe(({ spaces }) => {
         result.wake(spaces);
       });
-      afterTest(() => query.close());
+      onTestFinished(() => query.close());
       expect(await result.wait()).to.be.length(0);
 
       result.reset();