matrix-js-sdk 42.3.0 → 42.4.0

package/src/rust-crypto/index.ts

@@ -103,6 +103,25 @@ export interface InitRustCryptoArgs {
      * considered verified, without any manual verification taking place.
      */
     caCertsPem?: string;
+
+    /**
+     * Optional async function for signing some data with an X.509 certificate.
+     * Used to sign the user's identity so compatible clients will recognise
+     * this user as verified without manual verification taking place. If you
+     * supply this you must also supply {@link InitRustCryptoArgs#x509Validity}.
+     */
+    x509Signer?: (item: Uint8Array) => Promise<{
+        signature_bytes: Uint8Array;
+        certificate_chain: string;
+        signature_scheme: "RsaPssSha512";
+    }>;
+
+    /**
+     * Optional function returning the validity period of the X.509 certificate
+     * used for signing, as the number of milliseconds since the Unix epoch. If
+     * you supply this you must also supply {@link InitRustCryptoArgs#x509Signer}.
+     */
+    x509Validity?: () => number;
 }
 
 /**
@@ -158,6 +177,8 @@ async function initOlmMachine(
         legacyCryptoStore,
         enableEncryptedStateEvents,
         caCertsPem,
+        x509Signer,
+        x509Validity,
     }: InitRustCryptoArgs,
     storeHandle: StoreHandle,
 ): Promise<RustCrypto> {
@@ -169,6 +190,8 @@ async function initOlmMachine(
         storeHandle,
         logger,
         caCertsPem,
+        x509Signer,
+        x509Validity,
     );
 
     // A final migration step, now that we have an OlmMachine.

package/src/rust-crypto/rust-crypto.ts

@@ -19,7 +19,7 @@ import * as RustSdkCryptoJs from "@matrix-org/matrix-sdk-crypto-wasm";
 
 import type { IMegolmSessionData } from "../@types/crypto.ts";
 import { KnownMembership } from "../@types/membership.ts";
-import { type IDeviceLists, type IToDeviceEvent, type ReceivedToDeviceMessage } from "../sync-accumulator.ts";
+import { type IToDeviceEvent, type ReceivedToDeviceMessage } from "../sync-accumulator.ts";
 import type { ToDeviceBatch, ToDevicePayload } from "../models/ToDeviceMessage.ts";
 import { type MatrixEvent, MatrixEventEvent } from "../models/event.ts";
 import { type Room } from "../models/room.ts";
@@ -30,6 +30,7 @@ import {
     DecryptionError,
     type EventDecryptionResult,
     type OnSyncCompletedData,
+    type SyncCryptoChanges,
 } from "../common-crypto/CryptoBackend.ts";
 import { type Logger, LogSpan } from "../logger.ts";
 import { type IHttpOpts, type MatrixHttpApi, Method } from "../http-api/index.ts";
@@ -292,12 +293,12 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, CryptoEventH
     public async decryptEvent(event: MatrixEvent): Promise<EventDecryptionResult> {
         const roomId = event.getRoomId();
         if (!roomId) {
-            // presumably, a to-device message. These are normally decrypted in preprocessToDeviceMessages
+            // presumably, a to-device message. These are normally decrypted in processSyncChanges
             // so the fact it has come back here suggests that decryption failed.
             //
             // once we drop support for the libolm crypto implementation, we can stop passing to-device messages
             // through decryptEvent and hence get rid of this case.
-            throw new Error("to-device event was not decrypted in preprocessToDeviceMessages");
+            throw new Error("to-device event was not decrypted in processSyncChanges");
         }
         return await this.eventDecryptor.attemptEventDecryption(event, this.deviceIsolationMode);
     }
@@ -1691,41 +1692,31 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, CryptoEventH
     ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
     /**
-     * Apply sync changes to the olm machine
-     * @param events - the received to-device messages
-     * @param oneTimeKeysCounts - the received one time key counts
-     * @param unusedFallbackKeys - the received unused fallback keys
-     * @param devices - the received device list updates
-     * @returns A list of processed to-device messages.
+     * Implementation of {@link SyncCryptoCallbacks.processSyncChanges}.
+     *
+     * Passes all of the encryption-relevant data from a sync response to the OlmMachine in a single call, and
+     * post-processes the resulting to-device messages.
      */
-    private async receiveSyncChanges({
-        events,
-        oneTimeKeysCounts = new Map<string, number>(),
+    public async processSyncChanges({
+        toDeviceEvents,
+        deviceLists,
+        oneTimeKeysCounts,
         unusedFallbackKeys,
-        devices = new RustSdkCryptoJs.DeviceLists(),
-    }: {
-        events?: IToDeviceEvent[];
-        oneTimeKeysCounts?: Map<string, number>;
-        unusedFallbackKeys?: Set<string>;
-        devices?: RustSdkCryptoJs.DeviceLists;
-    }): Promise<RustSdkCryptoJs.ProcessedToDeviceEvent[]> {
-        return await this.olmMachine.receiveSyncChanges(
-            events ? JSON.stringify(events) : "[]",
-            devices,
-            oneTimeKeysCounts,
-            unusedFallbackKeys,
+        useMsc4186 = false,
+    }: SyncCryptoChanges): Promise<ReceivedToDeviceMessage[]> {
+        const events = JSON.stringify(toDeviceEvents);
+        const devices = new RustSdkCryptoJs.DeviceLists(
+            deviceLists?.changed?.map((userId) => new RustSdkCryptoJs.UserId(userId)),
+            deviceLists?.left?.map((userId) => new RustSdkCryptoJs.UserId(userId)),
         );
-    }
+        const counts = new Map(Object.entries(oneTimeKeysCounts ?? {}));
+        const fallbackKeys = unusedFallbackKeys && new Set(unusedFallbackKeys);
 
-    /** called by the sync loop to preprocess incoming to-device messages
-     *
-     * @param events - the received to-device messages
-     * @returns A list of preprocessed to-device messages.
-     */
-    public async preprocessToDeviceMessages(events: IToDeviceEvent[]): Promise<ReceivedToDeviceMessage[]> {
-        // send the received to-device messages into receiveSyncChanges. We have no info on device-list changes,
-        // one-time-keys, or fallback keys, so just pass empty data.
-        const processed = await this.receiveSyncChanges({ events });
+        // The two variants differ only in how they treat a missing one-time key count: zero keys on the server (sync
+        // v2) versus unchanged (sliding sync).
+        const processed = useMsc4186
+            ? await this.olmMachine.receiveSyncChangesMsc4186(events, devices, counts, fallbackKeys)
+            : await this.olmMachine.receiveSyncChanges(events, devices, counts, fallbackKeys);
 
         const received: ReceivedToDeviceMessage[] = [];
 
@@ -1808,39 +1799,6 @@ export class RustCrypto extends TypedEventEmitter<RustCryptoEvents, CryptoEventH
         return received;
     }
 
-    /** called by the sync loop to process one time key counts and unused fallback keys
-     *
-     * @param oneTimeKeysCounts - the received one time key counts
-     * @param unusedFallbackKeys - the received unused fallback keys
-     */
-    public async processKeyCounts(
-        oneTimeKeysCounts?: Record<string, number>,
-        unusedFallbackKeys?: string[],
-    ): Promise<void> {
-        const mapOneTimeKeysCount = oneTimeKeysCounts && new Map<string, number>(Object.entries(oneTimeKeysCounts));
-        const setUnusedFallbackKeys = unusedFallbackKeys && new Set<string>(unusedFallbackKeys);
-
-        if (mapOneTimeKeysCount !== undefined || setUnusedFallbackKeys !== undefined) {
-            await this.receiveSyncChanges({
-                oneTimeKeysCounts: mapOneTimeKeysCount,
-                unusedFallbackKeys: setUnusedFallbackKeys,
-            });
-        }
-    }
-
-    /** called by the sync loop to process the notification that device lists have
-     * been changed.
-     *
-     * @param deviceLists - device_lists field from /sync
-     */
-    public async processDeviceLists(deviceLists: IDeviceLists): Promise<void> {
-        const devices = new RustSdkCryptoJs.DeviceLists(
-            deviceLists.changed?.map((userId) => new RustSdkCryptoJs.UserId(userId)),
-            deviceLists.left?.map((userId) => new RustSdkCryptoJs.UserId(userId)),
-        );
-        await this.receiveSyncChanges({ devices });
-    }
-
     /** called by the sync loop on m.room.encryption events
      *
      * @param room - in which the event was received

package/src/sliding-sync-sdk.ts

@@ -28,7 +28,7 @@ import {
     defaultClientOpts,
     defaultSyncApiOpts,
     type SetPresence,
-    processToDeviceMessages,
+    processSyncCryptoChanges,
 } from "./sync.ts";
 import { type MatrixEvent } from "./models/event.ts";
 import {
@@ -39,7 +39,7 @@ import {
     type IStickyStateEvent,
     type IStrippedState,
     type ISyncResponse,
-    type ReceivedToDeviceMessage,
+    type IToDeviceEvent,
 } from "./sync-accumulator.ts";
 import { MatrixError } from "./http-api/index.ts";
 import {
@@ -74,8 +74,63 @@ type ExtensionE2EEResponse = Pick<
     | "org.matrix.msc2732.device_unused_fallback_key_types"
 >;
 
+/**
+ * Collects the encryption-relevant parts of a sliding sync response, which arrive via two separate extensions
+ * (`e2ee` and `to_device`), so that they can be passed to the crypto layer in a single call once the whole response
+ * has been processed. See {@link SyncCryptoCallbacks.processSyncChanges} for why this matters.
+ */
+class E2EESyncChangesCollector {
+    private toDeviceEvents: IToDeviceEvent[] = [];
+    private e2ee?: ExtensionE2EEResponse;
+    private hasChanges = false;
+
+    public constructor(
+        private readonly client: MatrixClient,
+        private readonly cryptoCallbacks?: SyncCryptoCallbacks,
+    ) {}
+
+    public onToDeviceEvents(events: IToDeviceEvent[]): void {
+        this.toDeviceEvents = events;
+        this.hasChanges = true;
+    }
+
+    public onE2EEChanges(data: ExtensionE2EEResponse): void {
+        this.e2ee = data;
+        this.hasChanges = true;
+    }
+
+    /**
+     * Pass the collected changes to the crypto layer, and emit the resulting to-device messages on the client.
+     *
+     * A no-op if nothing has been collected since the last flush, so it is safe to call once per extension.
+     */
+    public async flush(): Promise<void> {
+        if (!this.hasChanges) return;
+        const toDeviceEvents = this.toDeviceEvents;
+        const e2ee = this.e2ee;
+        this.toDeviceEvents = [];
+        this.e2ee = undefined;
+        this.hasChanges = false;
+
+        // Fields omitted from the `e2ee` extension are unchanged since the last response; the crypto layer knows to
+        // interpret them that way given `useMsc4186`.
+        await processSyncCryptoChanges(this.client, this.cryptoCallbacks, {
+            toDeviceEvents,
+            deviceLists: e2ee?.device_lists,
+            oneTimeKeysCounts: e2ee?.device_one_time_keys_count,
+            unusedFallbackKeys:
+                e2ee?.device_unused_fallback_key_types ?? e2ee?.["org.matrix.msc2732.device_unused_fallback_key_types"],
+            useMsc4186: true,
+        });
+        this.cryptoCallbacks?.onSyncCompleted({});
+    }
+}
+
 class ExtensionE2EE implements Extension<ExtensionE2EERequest, ExtensionE2EEResponse> {
-    public constructor(private readonly crypto: SyncCryptoCallbacks) {}
+    public constructor(
+        private readonly crypto: SyncCryptoCallbacks,
+        private readonly collector: E2EESyncChangesCollector,
+    ) {}
 
     public name(): string {
         return "e2ee";
@@ -102,18 +157,11 @@ class ExtensionE2EE implements Extension<ExtensionE2EERequest, ExtensionE2EEResp
     }
 
     public async onResponse(data: ExtensionE2EEResponse): Promise<void> {
-        // Handle device list updates
-        if (data.device_lists) {
-            await this.crypto.processDeviceLists(data.device_lists);
-        }
-
-        // Handle one_time_keys_count and unused_fallback_key_types
-        await this.crypto.processKeyCounts(
-            data.device_one_time_keys_count,
-            data["device_unused_fallback_key_types"] || data["org.matrix.msc2732.device_unused_fallback_key_types"],
-        );
+        this.collector.onE2EEChanges(data);
+    }
 
-        this.crypto.onSyncCompleted({});
+    public async onResponseComplete(): Promise<void> {
+        await this.collector.flush();
     }
 }
 
@@ -131,10 +179,7 @@ type ExtensionToDeviceResponse = {
 class ExtensionToDevice implements Extension<ExtensionToDeviceRequest, ExtensionToDeviceResponse> {
     private nextBatch: string | null = null;
 
-    public constructor(
-        private readonly client: MatrixClient,
-        private readonly cryptoCallbacks?: SyncCryptoCallbacks,
-    ) {}
+    public constructor(private readonly collector: E2EESyncChangesCollector) {}
 
     public name(): string {
         return "to_device";
@@ -153,21 +198,13 @@ class ExtensionToDevice implements Extension<ExtensionToDeviceRequest, Extension
     }
 
     public async onResponse(data: ExtensionToDeviceResponse): Promise<void> {
-        const events = data["events"] || [];
-        let receivedToDeviceMessages: ReceivedToDeviceMessage[];
-        if (this.cryptoCallbacks) {
-            receivedToDeviceMessages = await this.cryptoCallbacks.preprocessToDeviceMessages(events);
-        } else {
-            // Crypto is not enabled, so we just return the events.
-            receivedToDeviceMessages = events.map((rawEvent) => ({
-                message: rawEvent,
-                encryptionInfo: null,
-            }));
-        }
-        processToDeviceMessages(receivedToDeviceMessages, this.client);
-
+        this.collector.onToDeviceEvents(data["events"] || []);
         this.nextBatch = data.next_batch;
     }
+
+    public async onResponseComplete(): Promise<void> {
+        await this.collector.flush();
+    }
 }
 
 type ExtensionAccountDataRequest = {
@@ -408,15 +445,18 @@ export class SlidingSyncSdk {
 
         this.slidingSync.on(SlidingSyncEvent.Lifecycle, this.onLifecycle.bind(this));
         this.slidingSync.on(SlidingSyncEvent.RoomData, this.onRoomData.bind(this));
+        // The `e2ee` and `to_device` extensions feed a shared collector, so that the crypto layer sees all the
+        // encryption-relevant data from a response in a single call.
+        const e2eeCollector = new E2EESyncChangesCollector(this.client, this.syncOpts.cryptoCallbacks);
         const extensions: Extension<any, any>[] = [
-            new ExtensionToDevice(this.client, this.syncOpts.cryptoCallbacks),
+            new ExtensionToDevice(e2eeCollector),
             new ExtensionAccountData(this.client),
             new ExtensionTyping(this.client),
             new ExtensionReceipts(this.client),
             new ExtensionStickyEvents(this.client),
         ];
         if (this.syncOpts.cryptoCallbacks) {
-            extensions.push(new ExtensionE2EE(this.syncOpts.cryptoCallbacks));
+            extensions.push(new ExtensionE2EE(this.syncOpts.cryptoCallbacks, e2eeCollector));
         }
         extensions.forEach((ext) => {
             this.slidingSync.registerExtension(ext);

package/src/sliding-sync.ts

@@ -249,6 +249,15 @@ export interface Extension<Req extends object, Res extends object> {
      * @returns The state when it should be called.
      */
     when(): ExtensionState;
+    /**
+     * An optional function which is called once per sync response, after `onResponse` has been called on all of
+     * the extensions (of the same `when()` stage) which were present in the response. It is called even if this
+     * extension was not itself present in the response.
+     *
+     * This allows extensions which need to combine their data with that of other extensions (such as `e2ee` and
+     * `to_device`) to act once the whole response has been seen.
+     */
+    onResponseComplete?(): Promise<void>;
 }
 
 /**
@@ -487,23 +496,31 @@ export class SlidingSync extends TypedEventEmitter<SlidingSyncEvent, SlidingSync
     }
 
     private async onPreExtensionsResponse(ext: Record<string, object>): Promise<void> {
-        await Promise.all(
-            Object.keys(ext).map(async (extName) => {
-                if (this.extensions[extName].when() == ExtensionState.PreProcess) {
-                    await this.extensions[extName].onResponse(ext[extName]);
-                }
-            }),
-        );
+        await this.onExtensionsResponse(ext, ExtensionState.PreProcess);
     }
 
     private async onPostExtensionsResponse(ext: Record<string, object>): Promise<void> {
-        await Promise.all(
-            Object.keys(ext).map(async (extName) => {
-                if (this.extensions[extName].when() == ExtensionState.PostProcess) {
-                    await this.extensions[extName].onResponse(ext[extName]);
-                }
-            }),
-        );
+        await this.onExtensionsResponse(ext, ExtensionState.PostProcess);
+    }
+
+    /**
+     * Call `onResponse` on each registered extension of the given stage which is present in the response, and then
+     * `onResponseComplete` on every registered extension of that stage.
+     *
+     * Extensions are processed sequentially, in registration order, so that extensions whose data must be combined
+     * (such as `e2ee` and `to_device`) see a deterministic order.
+     */
+    private async onExtensionsResponse(ext: Record<string, object>, state: ExtensionState): Promise<void> {
+        const extensions = Object.values(this.extensions).filter((extension) => extension.when() === state);
+        for (const extension of extensions) {
+            const data = ext[extension.name()];
+            if (data !== undefined) {
+                await extension.onResponse(data);
+            }
+        }
+        for (const extension of extensions) {
+            await extension.onResponseComplete?.();
+        }
     }
 
     /**

package/src/sync.ts

@@ -23,7 +23,7 @@ limitations under the License.
  * for HTTP and WS at some point.
  */
 
-import type { SyncCryptoCallbacks } from "./common-crypto/CryptoBackend.ts";
+import type { SyncCryptoCallbacks, SyncCryptoChanges } from "./common-crypto/CryptoBackend.ts";
 import { type SyncUserProfile, User } from "./models/user.ts";
 import { NotificationCountType, Room, RoomEvent } from "./models/room.ts";
 import { deepCopy, noUnsafeEventProps, unsafeProp } from "./utils.ts";
@@ -1185,24 +1185,29 @@ export class SyncApi {
             }
         }
 
-        // handle to-device events
-        if (data.to_device && Array.isArray(data.to_device.events) && data.to_device.events.length > 0) {
-            const toDeviceMessages: IToDeviceEvent[] = data.to_device.events.filter(noUnsafeEventProps);
-
-            let receivedToDeviceMessages: ReceivedToDeviceMessage[];
-            if (this.syncOpts.cryptoCallbacks) {
-                receivedToDeviceMessages =
-                    await this.syncOpts.cryptoCallbacks.preprocessToDeviceMessages(toDeviceMessages);
-            } else {
-                // Crypto is not enabled, so we just return the events.
-                receivedToDeviceMessages = toDeviceMessages.map((rawEvent) => ({
-                    message: rawEvent,
-                    encryptionInfo: null,
-                }));
-            }
+        // Handle to-device events, device list changes, one-time key counts and unused fallback keys.
+        //
+        // These are passed to the crypto layer together, in a single call: see the documentation of
+        // `SyncCryptoCallbacks.processSyncChanges` for why they must not be split up. This has to happen before we
+        // process the room events, so that any room keys received in to-device messages can be used to decrypt them.
+        const toDeviceEvents: IToDeviceEvent[] = Array.isArray(data.to_device?.events) ? data.to_device.events : [];
+
+        // A cached sync (see `syncFromCache`) carries no E2EE data at all, so we skip the crypto layer for it: an
+        // absent `device_one_time_keys_count` would otherwise be taken to mean that there are no one-time keys on the
+        // server, triggering a spurious key upload on every restart.
+        if (!syncEventData.fromCache) {
+            await processSyncCryptoChanges(client, this.syncOpts.cryptoCallbacks, {
+                toDeviceEvents,
+                deviceLists: data.device_lists,
+                // Per the spec, an absent `device_one_time_keys_count` means there are no one-time keys on the server.
+                oneTimeKeysCounts: data.device_one_time_keys_count ?? {},
+                unusedFallbackKeys:
+                    data.device_unused_fallback_key_types ??
+                    data["org.matrix.msc2732.device_unused_fallback_key_types"],
+            });
+        }
 
-            processToDeviceMessages(receivedToDeviceMessages, client);
-        } else {
+        if (toDeviceEvents.length === 0) {
             // no more to-device events: we can stop polling with a short timeout.
             this.catchingUp = false;
         }
@@ -1567,23 +1572,6 @@ export class SyncApi {
                 client.getNotifTimelineSet()?.addLiveEvent(event, { addToState: true });
             });
         }
-
-        // Handle device list updates
-        if (data.device_lists) {
-            if (this.syncOpts.cryptoCallbacks) {
-                await this.syncOpts.cryptoCallbacks.processDeviceLists(data.device_lists);
-            } else {
-                // FIXME if we *don't* have a crypto module, we still need to
-                // invalidate the device lists. But that would require a
-                // substantial bit of rework :/.
-            }
-        }
-
-        // Handle one_time_keys_count and unused fallback keys
-        await this.syncOpts.cryptoCallbacks?.processKeyCounts(
-            data.device_one_time_keys_count,
-            data.device_unused_fallback_key_types ?? data["org.matrix.msc2732.device_unused_fallback_key_types"],
-        );
     }
 
     /**
@@ -2001,6 +1989,41 @@ export function _createAndReEmitRoom(client: MatrixClient, roomId: string, opts:
     return room;
 }
 
+/**
+ * Pass the encryption-relevant parts of a sync response to the crypto layer, and dispatch the resulting to-device
+ * messages on the client.
+ *
+ * `changes.toDeviceEvents` is first filtered with {@link noUnsafeEventProps}. If crypto is not enabled, the to-device
+ * messages are dispatched as received.
+ */
+export async function processSyncCryptoChanges(
+    client: MatrixClient,
+    cryptoCallbacks: SyncCryptoCallbacks | undefined,
+    changes: SyncCryptoChanges,
+): Promise<void> {
+    const toDeviceEvents = changes.toDeviceEvents.filter(noUnsafeEventProps);
+
+    let receivedToDeviceMessages: ReceivedToDeviceMessage[];
+    if (cryptoCallbacks) {
+        try {
+            receivedToDeviceMessages = await cryptoCallbacks.processSyncChanges({ ...changes, toDeviceEvents });
+        } catch (e) {
+            // Don't let a failure in the crypto layer stop the rest of the sync response from being processed: the
+            // sync token has already been advanced, so the room data would otherwise be lost.
+            logger.error("Error passing sync changes to the crypto layer", e);
+            return;
+        }
+    } else {
+        // Crypto is not enabled, so we just return the events.
+        //
+        // FIXME if we *don't* have a crypto module, we still need to invalidate the device lists. But that would
+        // require a substantial bit of rework :/.
+        receivedToDeviceMessages = toDeviceEvents.map((message) => ({ message, encryptionInfo: null }));
+    }
+
+    processToDeviceMessages(receivedToDeviceMessages, client);
+}
+
 /**
  * Process a list of (decrypted, where possible) received to-device events.
  *