matrix-nodejs-homeserver 0.3.3 → 0.3.5

package/dist/api/slidingsync.js

@@ -6,6 +6,7 @@ import { getState } from "../core/authrules.js";
 import { record, string, unknown } from "../vendor/zod/v4/classic/schemas.js";
 import { body, lenient } from "../http/input.js";
 import { procedure } from "../http/procedure.js";
+import { longPoll } from "./longpoll.js";
 import { roomEphemeral, roomTimeline, toStripped } from "./sync.js";
 //#region src/api/slidingsync.ts
 const slidingSyncBody = body({
@@ -104,37 +105,34 @@ async function slidingSync(ctx) {
 			rooms: {}
 		};
 	}
-	const deadline = Date.now() + timeout;
-	for (;;) {
-		const to = await hs.streams.position();
-		const { response, empty, next } = await buildResponse(hs, s, body, conn, since, to);
-		const remaining = deadline - Date.now();
-		if (since === null || !empty || remaining <= 0) {
-			await hs.db.upsert("sliding_sync_connections", {
-				user_id: s.userId,
-				device_id: s.deviceId,
-				conn_id: connId,
-				pos: to,
-				state: {
-					...next,
-					prevPos: since
-				},
-				updated_ts: Date.now()
-			});
-			return {
-				pos: String(to),
-				...response
-			};
-		}
-		if (await hs.streams.position() > to) continue;
-		await hs.notifier.wait(s.userId, remaining, ctx.request.signal);
-		if (ctx.request.signal?.aborted) return {
-			pos: String(since),
-			lists: {},
-			rooms: {},
-			extensions: {}
+	const { value, to, aborted } = await longPoll(hs, s.userId, timeout, ctx.request.signal, async (to) => {
+		const built = await buildResponse(hs, s, body, conn, since, to);
+		return {
+			value: built,
+			done: since === null || !built.empty
 		};
-	}
+	});
+	if (aborted && value.empty) return {
+		pos: String(since),
+		lists: {},
+		rooms: {},
+		extensions: {}
+	};
+	await hs.db.upsert("sliding_sync_connections", {
+		user_id: s.userId,
+		device_id: s.deviceId,
+		conn_id: connId,
+		pos: to,
+		state: {
+			...value.next,
+			prevPos: since
+		},
+		updated_ts: Date.now()
+	});
+	return {
+		pos: String(to),
+		...value.response
+	};
 }
 async function buildResponse(hs, s, body, conn, since, to) {
 	const userId = s.userId;

package/dist/api/sync.js

@@ -8,9 +8,11 @@ import { deviceListChangesQuery } from "../core/keys.js";
 import { body } from "../http/input.js";
 import { procedure } from "../http/procedure.js";
 import { readBatch } from "../storage/batch.js";
+import { trackedReads } from "../core/changes.js";
+import { fullProfile } from "../core/profile.js";
 import { matchesFilter, roomAllowed, validateSyncFilter } from "./filters.js";
+import { longPoll } from "./longpoll.js";
 import { Semaphore } from "../core/semaphore.js";
-import { fullProfile } from "../core/profile.js";
 //#region src/api/sync.ts
 function parseSyncToken(raw) {
 	if (!raw) return null;
@@ -330,6 +332,8 @@ async function changesSince(hs, p, memberships, since, accountRows) {
 		operator: "lte",
 		value: p.to
 	}];
+	const logs = hs.changes?.logs;
+	const mayHaveChanged = (log, rooms) => logs ? logs[log].filterChanged(rooms, since) : rooms;
 	const chunks = (list) => Array.from({ length: Math.ceil(list.length / 500) }, (_, i) => list.slice(i * 500, i * 500 + 500));
 	const inRooms = (chunk) => ({
 		field: "room_id",
@@ -337,31 +341,34 @@ async function changesSince(hs, p, memberships, since, accountRows) {
 		value: chunk
 	});
 	const queries = [];
-	for (const chunk of chunks(joined)) {
-		queries.push({
-			model: "events",
-			opts: {
-				where: [inRooms(chunk), ...range],
-				fields: ["room_id"]
-			}
-		});
-		queries.push({
-			model: "receipts",
-			opts: { where: [inRooms(chunk), ...range] }
-		});
-		queries.push({
-			model: "partial_state_rooms",
-			opts: {
-				where: [inRooms(chunk), {
-					field: "completed_stream",
-					operator: "gt",
-					value: since
-				}],
-				fields: ["room_id"]
-			}
-		});
-	}
-	for (const chunk of chunks(anyMembership)) queries.push({
+	const kinds = [];
+	const add = (kind, query) => {
+		kinds.push(kind);
+		queries.push(query);
+	};
+	for (const chunk of chunks(mayHaveChanged("roomEvents", joined))) add("events", {
+		model: "events",
+		opts: {
+			where: [inRooms(chunk), ...range],
+			fields: ["room_id"]
+		}
+	});
+	for (const chunk of chunks(mayHaveChanged("roomReceipts", joined))) add("receipts", {
+		model: "receipts",
+		opts: { where: [inRooms(chunk), ...range] }
+	});
+	for (const chunk of chunks(mayHaveChanged("roomPartialState", joined))) add("completed", {
+		model: "partial_state_rooms",
+		opts: {
+			where: [inRooms(chunk), {
+				field: "completed_stream",
+				operator: "gt",
+				value: since
+			}],
+			fields: ["room_id"]
+		}
+	});
+	for (const chunk of chunks(mayHaveChanged("roomMembers", anyMembership))) add("members", {
 		model: "memberships",
 		opts: { where: [inRooms(chunk), {
 			field: "stream",
@@ -369,20 +376,18 @@ async function changesSince(hs, p, memberships, since, accountRows) {
 			value: since
 		}] }
 	});
-	const results = await readBatch(hs.db, queries);
+	const results = queries.length ? await readBatch(hs.db, queries) : [];
 	const touched = /* @__PURE__ */ new Set();
 	const receipts = /* @__PURE__ */ new Map();
-	const joinedChunks = Math.ceil(joined.length / 500);
-	for (let c = 0; c < joinedChunks; c++) {
-		const [events, receiptRows, completed] = results.slice(c * 3, c * 3 + 3);
-		for (const r of events) touched.add(r.room_id);
-		for (const r of receiptRows) {
-			touched.add(r.room_id);
-			receipts.set(r.room_id, [...receipts.get(r.room_id) ?? [], r]);
+	const memberChanges = [];
+	results.forEach((rows, i) => {
+		const kind = kinds[i];
+		for (const r of rows) {
+			if (kind === "members") memberChanges.push(r);
+			else touched.add(r.room_id);
+			if (kind === "receipts") receipts.set(r.room_id, [...receipts.get(r.room_id) ?? [], r]);
 		}
-		for (const r of completed) touched.add(r.room_id);
-	}
-	const memberChanges = results.slice(joinedChunks * 3).flat();
+	});
 	const accountData = /* @__PURE__ */ new Map();
 	for (const r of accountRows) {
 		if (r.room_id) touched.add(r.room_id);
@@ -428,76 +433,115 @@ async function buildSync(hs, p) {
 		operator: "lte",
 		value: p.since.toDevice
 	}]) : Promise.resolve(0);
-	const [memberships, ignoredRows, toDeviceRows, oneTimeKeyRows, fallbackRows, accountRows = []] = await readBatch(hs.db, [
-		{
-			model: "memberships",
-			opts: { where: [{
-				field: "user_id",
-				value: userId
-			}] }
-		},
+	const device = `${userId}|${p.session.deviceId}`;
+	const [memberships, ignoredRows, toDeviceRows, oneTimeKeyRows, fallbackRows, accountRows = []] = await trackedReads(hs.db, hs.changes, [
 		{
-			model: "account_data",
-			opts: { where: [
-				{
+			query: {
+				model: "memberships",
+				opts: { where: [{
 					field: "user_id",
 					value: userId
-				},
-				{
-					field: "room_id",
-					value: ""
-				},
-				{
-					field: "type",
-					value: M.IGNORED_USER_LIST
-				}
-			] }
+				}] }
+			},
+			cached: {
+				name: "userMemberships",
+				key: userId
+			}
 		},
 		{
-			model: "to_device",
-			opts: {
-				where: [...deviceWhere, {
-					field: "stream",
-					operator: "gt",
-					value: p.since?.toDevice ?? 0
-				}],
-				sortBy: {
-					field: "stream",
-					direction: "asc"
-				},
-				limit: 100
+			query: {
+				model: "account_data",
+				opts: { where: [
+					{
+						field: "user_id",
+						value: userId
+					},
+					{
+						field: "room_id",
+						value: ""
+					},
+					{
+						field: "type",
+						value: M.IGNORED_USER_LIST
+					}
+				] }
+			},
+			cached: {
+				name: "ignoredUsers",
+				key: userId
 			}
 		},
 		{
-			model: "one_time_keys",
-			opts: {
-				where: [...deviceWhere, {
-					field: "fallback",
-					value: false
-				}],
-				fields: ["algorithm"]
+			query: {
+				model: "to_device",
+				opts: {
+					where: [...deviceWhere, {
+						field: "stream",
+						operator: "gt",
+						value: p.since?.toDevice ?? 0
+					}],
+					sortBy: {
+						field: "stream",
+						direction: "asc"
+					},
+					limit: 100
+				}
+			},
+			unlessChanged: {
+				log: "toDevice",
+				key: device,
+				after: p.since?.toDevice ?? 0
 			}
 		},
 		{
-			model: "fallback_keys",
-			opts: {
-				where: [...deviceWhere, {
-					field: "used",
-					value: false
-				}],
-				fields: ["algorithm"]
+			query: {
+				model: "one_time_keys",
+				opts: {
+					where: [...deviceWhere, {
+						field: "fallback",
+						value: false
+					}],
+					fields: ["algorithm"]
+				}
+			},
+			cached: {
+				name: "oneTimeKeys",
+				key: device
+			}
+		},
+		{
+			query: {
+				model: "fallback_keys",
+				opts: {
+					where: [...deviceWhere, {
+						field: "used",
+						value: false
+					}],
+					fields: ["algorithm"]
+				}
+			},
+			cached: {
+				name: "fallbackKeys",
+				key: device
 			}
 		},
 		...incremental ? [{
-			model: "account_data",
-			opts: { where: [{
-				field: "user_id",
-				value: userId
-			}, {
-				field: "stream",
-				operator: "gt",
-				value: since
-			}] }
+			query: {
+				model: "account_data",
+				opts: { where: [{
+					field: "user_id",
+					value: userId
+				}, {
+					field: "stream",
+					operator: "gt",
+					value: since
+				}] }
+			},
+			unlessChanged: {
+				log: "accountData",
+				key: userId,
+				after: since
+			}
 		}] : []
 	]);
 	p = {
@@ -644,17 +688,31 @@ async function buildSync(hs, p) {
 			value: []
 		}] }
 	};
-	let [presenceRows = [], newlySharedPresence = [], deviceRows] = await readBatch(hs.db, [
-		presenceQuery,
-		{
+	let [presenceRows = [], newlySharedPresence = [], deviceRows] = await trackedReads(hs.db, hs.changes, [
+		since === null ? { query: presenceQuery } : {
+			query: presenceQuery,
+			unlessChanged: {
+				log: "presence",
+				key: "*",
+				after: since
+			}
+		},
+		{ query: {
 			model: "presence",
 			opts: { where: [{
 				field: "user_id",
 				operator: "in",
 				value: [...newlyShared]
 			}] }
-		},
-		...since !== null ? [recent("device_list_changes")] : []
+		} },
+		...since !== null ? [{
+			query: recent("device_list_changes"),
+			unlessChanged: {
+				log: "deviceLists",
+				key: "*",
+				after: since
+			}
+		}] : []
 	]);
 	if (since !== null) {
 		if (presenceRows.length > RECENT_CHANGES) presenceRows = await hs.db.findMany("presence", { where: [{
@@ -745,14 +803,12 @@ async function sync(ctx) {
 	const timeout = Math.min(Math.max(Number.parseInt(ctx.query.get("timeout") ?? "0", 10) || 0, 0), hs.config.maxSyncTimeoutMs);
 	const setPresence = ctx.query.get("set_presence") ?? "online";
 	if (setPresence !== "offline") await touchPresence(hs, session.userId, setPresence);
-	const deadline = Date.now() + timeout;
-	for (;;) {
-		const to = await hs.streams.position();
-		const ignored = /* @__PURE__ */ new Set();
-		const stable = ctx.query.get("use_state_after") === "true";
-		const unstable = ctx.query.get(UNSTABLE.ORG.MATRIX.MSC4222.USE_STATE_AFTER) === "true";
-		const useStateAfter = stable || unstable;
-		const stateAfterKey = stable ? "state_after" : UNSTABLE.ORG.MATRIX.MSC4222.STATE_AFTER;
+	const ignored = /* @__PURE__ */ new Set();
+	const stable = ctx.query.get("use_state_after") === "true";
+	const unstable = ctx.query.get(UNSTABLE.ORG.MATRIX.MSC4222.USE_STATE_AFTER) === "true";
+	const useStateAfter = stable || unstable;
+	const stateAfterKey = stable ? "state_after" : UNSTABLE.ORG.MATRIX.MSC4222.STATE_AFTER;
+	const { value } = await longPoll(hs, session.userId, timeout, ctx.request.signal, async (to) => {
 		const { body, empty } = await syncSlots(hs).run(() => buildSync(hs, {
 			session,
 			ignored,
@@ -763,20 +819,21 @@ async function sync(ctx) {
 			useStateAfter,
 			stateAfterKey
 		}));
-		const remaining = deadline - Date.now();
-		if (!since || !empty || remaining <= 0 || fullState) return body;
-		if (await hs.streams.position() > to) continue;
-		await hs.notifier.wait(session.userId, remaining, ctx.request.signal);
-		if (ctx.request.signal?.aborted) return body;
-	}
+		return {
+			value: body,
+			done: !since || !empty || fullState
+		};
+	});
+	return value;
 }
 const LAST_ACTIVE_GRANULARITY_MS = 6e4;
 async function touchPresence(hs, userId, presence) {
 	const state = presence === "unavailable" ? "unavailable" : "online";
-	const row = await hs.db.findOne("presence", [{
+	const read = () => hs.db.findOne("presence", [{
 		field: "user_id",
 		value: userId
 	}]);
+	const row = hs.changes ? await hs.changes.values.presenceRow.get(userId, read) : await read();
 	if (row && row.presence === state) {
 		if (Date.now() - row.last_active_ts >= LAST_ACTIVE_GRANULARITY_MS) await hs.db.update("presence", [{
 			field: "user_id",
@@ -784,13 +841,17 @@ async function touchPresence(hs, userId, presence) {
 		}], { last_active_ts: Date.now() });
 		return;
 	}
-	await hs.streams.write((stream) => hs.db.upsert("presence", {
-		user_id: userId,
-		presence: state,
-		status_msg: row?.status_msg ?? null,
-		last_active_ts: Date.now(),
-		stream
-	}));
+	const stream = await hs.streams.write(async (stream) => {
+		await hs.db.upsert("presence", {
+			user_id: userId,
+			presence: state,
+			status_msg: row?.status_msg ?? null,
+			last_active_ts: Date.now(),
+			stream
+		});
+		return stream;
+	});
+	hs.notifier.notify([userId, ...await hs.rooms.usersSharingRooms(userId)], stream);
 	await hs.federation.onPresence(userId, state, row?.status_msg ?? null).catch(() => {});
 }
 function registerSyncRoutes(r) {

package/dist/api/threepid.js

@@ -89,7 +89,7 @@ function registerThreepidRoutes(r) {
 			notDelegated(ctx);
 			const s = await ctx.auth();
 			const body = ctx.input;
-			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId);
+			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId, { session: s });
 			const validated = await ctx.hs.threepids.validated({
 				sid: body.sid,
 				client_secret: body.client_secret

package/dist/api/uia.js

@@ -2,15 +2,26 @@ import { isObject } from "../core/json.js";
 import { randomString } from "../core/crypto.js";
 import { M, UNSTABLE } from "../core/m.js";
 import { MatrixError, errors } from "../core/errors.js";
+import { crossSigningResetAllowed } from "../core/oauthserver.js";
 //#region src/api/uia.ts
 const UIA_TTL_MS = 6e5;
 /**
 * User-interactive authentication. Resolves when one of `flows` has been
 * completed; otherwise throws the 401 response describing what is required.
-* `userId` binds password stages to an already-authenticated user.
+* `userId` binds password stages to an already-authenticated user. `session` is the caller's,
+* and `crossSigningReset` marks `/keys/device_signing/upload`, the one endpoint where tokens of
+* the OAuth 2.0 API may use UIA (the `m.oauth` stage).
 */
-async function requireUia(hs, body, flows, userId) {
-	if (hs.oauth && flows.some((f) => f.includes(M.LOGIN.PASSWORD))) flows = [[M.OAUTH], [UNSTABLE.ORG.MATRIX.CROSS_SIGNING_RESET]];
+async function requireUia(hs, body, flows, userId, opts = {}) {
+	const viaOAuth = !!opts.session?.oauth || !!hs.oauth && flows.some((f) => f.includes(M.LOGIN.PASSWORD));
+	let oauthUrl;
+	if (viaOAuth) {
+		const accountUri = hs.oauthServer?.accountManagementUri ?? hs.oauth?.options.accountManagementUrl;
+		if (!opts.crossSigningReset) throw new MatrixError(403, "M_FORBIDDEN", "Do this on the account management page", accountUri ? { account_management_uri: accountUri } : {});
+		if (userId && await crossSigningResetAllowed(hs, userId)) return { threepids: [] };
+		flows = [[M.OAUTH], [UNSTABLE.ORG.MATRIX.CROSS_SIGNING_RESET]];
+		oauthUrl = hs.oauthServer ? `${hs.oauthServer.accountManagementUri}?action=org.matrix.cross_signing_reset` : hs.oauth.options.accountManagementUrl ?? hs.oauth.options.issuer;
+	}
 	const auth = isObject(body.auth) ? body.auth : null;
 	let sessionId = typeof auth?.session === "string" ? auth.session : void 0;
 	const stored = sessionId ? await hs.cluster.get(`uia:${sessionId}`) : null;
@@ -25,8 +36,7 @@ async function requireUia(hs, body, flows, userId) {
 		userId: session.userId,
 		threepids: session.threepids
 	}, UIA_TTL_MS);
-	const oauthUrl = hs.oauth?.options.accountManagementUrl ?? hs.oauth?.options.issuer;
-	const params = hs.oauth ? {
+	const params = oauthUrl ? {
 		[M.OAUTH]: { url: oauthUrl },
 		[UNSTABLE.ORG.MATRIX.CROSS_SIGNING_RESET]: { url: oauthUrl }
 	} : {};
@@ -49,12 +59,14 @@ async function requireUia(hs, body, flows, userId) {
 			break;
 		case M.OAUTH:
 		case UNSTABLE.ORG.MATRIX.CROSS_SIGNING_RESET:
-			if (!hs.oauth) throw challenge({
+			if (!viaOAuth) throw challenge({
 				errcode: "M_UNRECOGNIZED",
 				error: "OAuth is not enabled"
 			});
-			session.completed.add(auth.type);
-			break;
+			throw challenge({
+				errcode: "M_FORBIDDEN",
+				error: "The reset has not been approved on the account page"
+			});
 		case M.LOGIN.REGISTRATION_TOKEN: {
 			const { registrationTokenValid } = await import("./account.js");
 			if (typeof auth.token !== "string" || !await registrationTokenValid(hs, auth.token)) throw challenge({

package/dist/api/user.js

@@ -185,7 +185,7 @@ function registerUserRoutes(r) {
 				value: ctx.params.deviceId
 			}];
 			if (!await ctx.hs.db.findOne("devices", where)) throw errors.notFound("Device not found");
-			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId);
+			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId, { session: s });
 			await ctx.hs.auth.deleteDevice(s.userId, ctx.params.deviceId);
 			return {};
 		}
@@ -196,7 +196,7 @@ function registerUserRoutes(r) {
 		handler: async (ctx) => {
 			const s = ctx.session;
 			const body = ctx.input;
-			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId);
+			await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId, { session: s });
 			for (const d of body.devices) if (typeof d === "string") await ctx.hs.auth.deleteDevice(s.userId, d);
 			return {};
 		}
@@ -275,7 +275,10 @@ function registerUserRoutes(r) {
 					"self_signing",
 					"user_signing"
 				].filter((type) => body[`${type}_key`] !== void 0).every((type) => stored.get(type) === canonicalJson(body[`${type}_key`]))) return {};
-				await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId);
+				await requireUia(ctx.hs, body, [[M.LOGIN.PASSWORD]], s.userId, {
+					session: s,
+					crossSigningReset: true
+				});
 			}
 			await ctx.hs.keys.uploadCrossSigning(s.userId, body);
 			return {};

package/dist/core/admin.js

@@ -301,6 +301,7 @@ var AdminService = class {
 			content: { tags: { [M.SERVER_NOTICE]: { order: .5 } } },
 			stream
 		}));
+		this.hs.notifier.notify([userId]);
 		return room_id;
 	}
 };

package/dist/core/auth.d.ts

@@ -13,6 +13,11 @@ export interface Session {
   suspended?: boolean;
   /** Set when the request was made with an application service token. */
   appservice?: AppServiceRegistration;
+  /**
+   * The token comes from the OAuth 2.0 API (this server's or a delegated one), so endpoints that
+   * need User-Interactive Authentication send the user to account management instead.
+   */
+  oauth?: boolean;
 }
 export interface CreatedSession {
   user_id: string;
@@ -20,6 +25,8 @@ export interface CreatedSession {
   access_token: string;
   refresh_token?: string;
   expires_in_ms?: number;
+  /** OAuth 2.0 API: the scope the tokens were issued for. */
+  scope?: string;
 }
 export declare class AuthService {
   private hs;
@@ -31,9 +38,33 @@ export declare class AuthService {
     deviceId?: string;
     displayName?: string;
     refreshToken?: boolean;
+    /** OAuth 2.0 API: the client the tokens are issued to. */
+    clientId?: string;
+    /** Access token lifetime; default `accessTokenLifetimeMs` (for tokens with a refresh token). */
+    lifetimeMs?: number;
+    /** OAuth 2.0 API: the scope granted. */
+    scope?: string;
+  }): Promise<CreatedSession>;
+  issueToken(userId: string, deviceId: string, withRefresh: boolean, opts?: {
+    clientId?: string | null;
+    lifetimeMs?: number;
+    parent?: string;
+    scope?: string | null;
+  }): Promise<CreatedSession>;
+  /**
+   * Exchange a refresh token for new tokens.
+   *
+   * https://github.com/k18k/matrix-nodejs-homeserver/issues/66: the old row used to be deleted
+   * at once, so a client whose response was lost could not retry and was logged out, and a
+   * refresh token used twice went unnoticed. As the OAuth 2.0 API asks (and the legacy API
+   * allows): the old refresh token keeps working until the new access token is first used; a
+   * retry before that replaces the unused tokens. Once the new token was used, the old refresh
+   * token is spent, and presenting it again means it leaked, so the device is logged out.
+   */
+  refresh(refreshToken: string, opts?: {
+    clientId?: string | null;
+    lifetimeMs?: number;
   }): Promise<CreatedSession>;
-  issueToken(userId: string, deviceId: string, withRefresh: boolean): Promise<CreatedSession>;
-  refresh(refreshToken: string): Promise<CreatedSession>;
   /** Drop cached sessions of a user (after admin/deactivation changes). */
   invalidateUser(userId: string): Promise<void>;
   /** Issue a short-lived single-use `m.login.token` for a user (SSO, or host-app sign-in). */