matrix-nodejs-homeserver 0.3.1 → 0.3.3

package/dist/api/sync.js

@@ -2,23 +2,27 @@ import { M, UNSTABLE } from "../core/m.js";
 import { eventIdOf, splitStateKey, stateKey, toClientEvent } from "../core/events.js";
 import { errors } from "../core/errors.js";
 import { getState } from "../core/authrules.js";
-import { loadRules } from "../core/pushrules.js";
+import { readOnlyRules } from "../core/pushrules.js";
 import { inviteDecision } from "../core/invitefilter.js";
+import { deviceListChangesQuery } from "../core/keys.js";
 import { body } from "../http/input.js";
 import { procedure } from "../http/procedure.js";
+import { readBatch } from "../storage/batch.js";
 import { matchesFilter, roomAllowed, validateSyncFilter } from "./filters.js";
+import { Semaphore } from "../core/semaphore.js";
 import { fullProfile } from "../core/profile.js";
 //#region src/api/sync.ts
 function parseSyncToken(raw) {
 	if (!raw) return null;
-	const m = /^s(\d+)(?:_(\d+))?$/.exec(raw);
+	const m = /^s(\d+)(?:_(\d+))?(_a)?$/.exec(raw);
 	if (!m) throw errors.invalidParam("Invalid since token");
 	return {
 		pos: Number(m[1]),
-		toDevice: Number(m[2] ?? m[1])
+		toDevice: Number(m[2] ?? m[1]),
+		ack: !!m[3]
 	};
 }
-const formatToken = (t) => `s${t.pos}_${t.toDevice}`;
+const formatToken = (t) => `s${t.pos}_${t.toDevice}${t.ack ? "_a" : ""}`;
 async function loadFilter(hs, userId, raw) {
 	if (!raw) return {};
 	if (raw.trim().startsWith("{")) {
@@ -64,7 +68,8 @@ async function snapshotAt(hs, roomId, pos) {
 			field: "stream",
 			direction: "desc"
 		},
-		limit: 1
+		limit: 1,
+		fields: ["state_snapshot"]
 	}))[0]?.state_snapshot ?? 0;
 }
 /** Build the timeline + state section of a room. */
@@ -84,6 +89,8 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 		operator: "gt",
 		value: since
 	});
+	const sinceIdsRead = since !== null ? snapshotAt(hs, roomId, since).then((id) => hs.rooms.snapshotIds(id)) : Promise.resolve({});
+	const scanned = [];
 	const events = [];
 	let limited = false;
 	let cursor = upTo;
@@ -102,6 +109,7 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 			limit: batchSize
 		});
 		if (rows.length === 0) break;
+		scanned.push(...rows);
 		const stored = rows.map((r) => hs.rooms.rowToStored(r));
 		const visible = await hs.rooms.filterVisible(p.session.userId, roomId, stored);
 		const ok = new Map(visible.map((v) => [v.pdu.event_id, v]));
@@ -126,7 +134,8 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 	}
 	if (limit === 0 && events.length === 0) limited = (await hs.db.findMany("events", {
 		where,
-		limit: 1
+		limit: 1,
+		fields: ["stream"]
 	})).length > 0;
 	if (since === null) events.sort((a, b) => (a.pdu.depth ?? 0) - (b.pdu.depth ?? 0) || a.stream - b.stream);
 	else events.reverse();
@@ -138,24 +147,50 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 		events.splice(0, lastGap);
 		limited = true;
 	}
-	if (!limited && events.length) limited = (await hs.db.findMany("events", {
-		where: [...where.filter((c) => c.operator !== "lte"), {
-			field: "stream",
-			operator: "lt",
-			value: events[0].stream
-		}],
-		limit: 1
-	})).length > 0;
 	const startPos = events.length ? Math.min(...events.map((e) => e.stream)) - 1 : upTo;
-	const startState = await hs.rooms.stateAtSnapshot(await snapshotAt(hs, roomId, startPos));
+	const newestAtOrBefore = async (pos) => {
+		const row = scanned.find((r) => r.stream <= pos);
+		if (row) return row;
+		return (await hs.db.findMany("events", {
+			where: [{
+				field: "room_id",
+				value: roomId
+			}, {
+				field: "stream",
+				operator: "lte",
+				value: pos
+			}],
+			sortBy: {
+				field: "stream",
+				direction: "desc"
+			},
+			limit: 1,
+			fields: ["stream", "state_snapshot"]
+		}))[0] ?? null;
+	};
+	const [atStart, atEnd] = await Promise.all([newestAtOrBefore(startPos), startPos === upTo ? null : newestAtOrBefore(upTo)]);
+	const startSnapshot = atStart?.state_snapshot ?? 0;
+	const endSnapshot = startPos === upTo ? startSnapshot : atEnd?.state_snapshot ?? 0;
+	if (!limited && events.length) {
+		if (events[0].stream === startPos + 1) limited = atStart !== null && (since === null || atStart.stream > since);
+		else limited = (await hs.db.findMany("events", {
+			where: [...where.filter((c) => c.operator !== "lte"), {
+				field: "stream",
+				operator: "lt",
+				value: events[0].stream
+			}],
+			limit: 1,
+			fields: ["stream"]
+		})).length > 0;
+	}
+	const [startState, endState] = await Promise.all([hs.rooms.stateAtSnapshot(startSnapshot), hs.rooms.stateAtSnapshot(endSnapshot)]);
 	const timelineIds = new Set(events.map((e) => e.pdu.event_id));
-	const endState = await hs.rooms.stateAtSnapshot(await snapshotAt(hs, roomId, upTo));
 	for (const [key, ev] of endState) {
 		if (timelineIds.has(ev.event_id) || startState.get(key)?.event_id === ev.event_id) continue;
 		if (events.some((e) => e.pdu.state_key !== void 0 && stateKey(e.pdu.type, e.pdu.state_key) === key)) continue;
 		startState.set(key, ev);
 	}
-	const sinceIds = since !== null ? await hs.rooms.snapshotIds(await snapshotAt(hs, roomId, since)) : {};
+	const sinceIds = await sinceIdsRead;
 	const sf = p.filter.room?.state;
 	const lazy = !!sf?.lazy_load_members;
 	const senders = new Set(events.map((e) => e.pdu.sender));
@@ -175,15 +210,12 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 		if (member && matchesFilter(sf, member)) state.push(member);
 	}
 	const stateAfter = [];
-	if (p.useStateAfter) {
-		const endState = await hs.rooms.stateAtSnapshot(await snapshotAt(hs, roomId, upTo));
-		for (const [key, ev] of endState) {
-			if (sinceIds[key] === ev.event_id) continue;
-			const [type, sk] = splitStateKey(key);
-			if (!matchesFilter(sf, ev)) continue;
-			if (lazy && type === M.ROOM.MEMBER && !senders.has(sk)) continue;
-			stateAfter.push(ev);
-		}
+	if (p.useStateAfter) for (const [key, ev] of endState) {
+		if (sinceIds[key] === ev.event_id) continue;
+		const [type, sk] = splitStateKey(key);
+		if (!matchesFilter(sf, ev)) continue;
+		if (lazy && type === M.ROOM.MEMBER && !senders.has(sk)) continue;
+		stateAfter.push(ev);
 	}
 	return {
 		stateAfter,
@@ -196,7 +228,7 @@ async function roomTimeline(hs, p, roomId, since, upTo) {
 		timelineEvents: events
 	};
 }
-async function roomEphemeral(hs, p, roomId) {
+async function roomEphemeral(hs, p, roomId, prefetched) {
 	const since = p.since?.pos ?? null;
 	const out = [];
 	const typing = hs.typing.typingSince(roomId, since);
@@ -213,7 +245,7 @@ async function roomEphemeral(hs, p, roomId) {
 		operator: "gt",
 		value: since
 	});
-	const receipts = (await hs.db.findMany("receipts", { where })).filter((r) => r.stream <= p.to);
+	const receipts = (prefetched ?? await hs.db.findMany("receipts", { where })).filter((r) => r.stream <= p.to);
 	const content = {};
 	for (const r of receipts) {
 		if (r.receipt_type === M.READ.PRIVATE && r.user_id !== p.session.userId) continue;
@@ -231,7 +263,7 @@ async function roomEphemeral(hs, p, roomId) {
 	});
 	return out.filter((e) => matchesFilter(p.filter.room?.ephemeral, e));
 }
-async function accountData(hs, p, roomId) {
+async function accountData(hs, p, roomId, prefetched) {
 	const where = [{
 		field: "user_id",
 		value: p.session.userId
@@ -244,7 +276,7 @@ async function accountData(hs, p, roomId) {
 		operator: "gt",
 		value: p.since.pos
 	});
-	const rows = await hs.db.findMany("account_data", { where });
+	const rows = prefetched ?? await hs.db.findMany("account_data", { where });
 	const filter = roomId ? p.filter.room?.account_data : p.filter.account_data;
 	return rows.filter((r) => r.stream <= p.to && (p.since || Object.keys(r.content ?? {}).length > 0)).map((r) => ({
 		type: r.type,
@@ -271,50 +303,223 @@ async function roomSummary(hs, roomId, userId) {
 	if (!getState(state, M.ROOM.NAME)?.content?.name && !getState(state, M.ROOM.CANONICAL_ALIAS)?.content?.alias) summary[M.HEROES] = heroes.sort();
 	return summary;
 }
+/**
+* What changed for the user since the token, for all of their rooms at once: the rooms with new
+* events or newly complete state, and the new receipts and account data by room.
+*
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/33: an incremental sync queried every
+* joined room (timeline, receipts, account data) and only then found that most had nothing new,
+* so a sync cost as many queries as the user had rooms, each time a long-poll woke up. These few
+* queries find the rooms that changed, and only those are built.
+*/
+/** One-time key counts by algorithm, as /sync reports them (signed_curve25519 always present). */
+function countAlgorithms(rows) {
+	const counts = { signed_curve25519: 0 };
+	for (const r of rows) counts[r.algorithm] = (counts[r.algorithm] ?? 0) + 1;
+	return counts;
+}
+async function changesSince(hs, p, memberships, since, accountRows) {
+	const joined = memberships.filter((m) => m.membership === "join").map((m) => m.room_id);
+	const anyMembership = [...new Set(memberships.map((m) => m.room_id))];
+	const range = [{
+		field: "stream",
+		operator: "gt",
+		value: since
+	}, {
+		field: "stream",
+		operator: "lte",
+		value: p.to
+	}];
+	const chunks = (list) => Array.from({ length: Math.ceil(list.length / 500) }, (_, i) => list.slice(i * 500, i * 500 + 500));
+	const inRooms = (chunk) => ({
+		field: "room_id",
+		operator: "in",
+		value: chunk
+	});
+	const queries = [];
+	for (const chunk of chunks(joined)) {
+		queries.push({
+			model: "events",
+			opts: {
+				where: [inRooms(chunk), ...range],
+				fields: ["room_id"]
+			}
+		});
+		queries.push({
+			model: "receipts",
+			opts: { where: [inRooms(chunk), ...range] }
+		});
+		queries.push({
+			model: "partial_state_rooms",
+			opts: {
+				where: [inRooms(chunk), {
+					field: "completed_stream",
+					operator: "gt",
+					value: since
+				}],
+				fields: ["room_id"]
+			}
+		});
+	}
+	for (const chunk of chunks(anyMembership)) queries.push({
+		model: "memberships",
+		opts: { where: [inRooms(chunk), {
+			field: "stream",
+			operator: "gt",
+			value: since
+		}] }
+	});
+	const results = await readBatch(hs.db, queries);
+	const touched = /* @__PURE__ */ new Set();
+	const receipts = /* @__PURE__ */ new Map();
+	const joinedChunks = Math.ceil(joined.length / 500);
+	for (let c = 0; c < joinedChunks; c++) {
+		const [events, receiptRows, completed] = results.slice(c * 3, c * 3 + 3);
+		for (const r of events) touched.add(r.room_id);
+		for (const r of receiptRows) {
+			touched.add(r.room_id);
+			receipts.set(r.room_id, [...receipts.get(r.room_id) ?? [], r]);
+		}
+		for (const r of completed) touched.add(r.room_id);
+	}
+	const memberChanges = results.slice(joinedChunks * 3).flat();
+	const accountData = /* @__PURE__ */ new Map();
+	for (const r of accountRows) {
+		if (r.room_id) touched.add(r.room_id);
+		accountData.set(r.room_id, [...accountData.get(r.room_id) ?? [], r]);
+	}
+	return {
+		touched,
+		receipts,
+		accountData,
+		memberChanges
+	};
+}
+/** Presence or device-list changes after a token read without a user filter, at most (#44). */
+const RECENT_CHANGES = 500;
+/** How many of a sync's joined rooms are built at once (#38). */
+const JOINED_ROOMS_AT_ONCE = 8;
+/** `fn` over `items` with at most `limit` running at once; results in input order. */
+async function mapConcurrently(items, limit, fn) {
+	const out = [];
+	let next = 0;
+	const worker = async () => {
+		while (next < items.length) {
+			const i = next++;
+			out[i] = await fn(items[i]);
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+	return out;
+}
 async function buildSync(hs, p) {
 	const userId = p.session.userId;
 	const since = p.since?.pos ?? null;
-	const memberships = await hs.db.findMany("memberships", { where: [{
+	const incremental = since !== null && !p.fullState;
+	const deviceWhere = [{
 		field: "user_id",
 		value: userId
-	}] });
+	}, {
+		field: "device_id",
+		value: p.session.deviceId
+	}];
+	const acknowledged = p.since?.ack ? hs.db.delete("to_device", [...deviceWhere, {
+		field: "stream",
+		operator: "lte",
+		value: p.since.toDevice
+	}]) : Promise.resolve(0);
+	const [memberships, ignoredRows, toDeviceRows, oneTimeKeyRows, fallbackRows, accountRows = []] = await readBatch(hs.db, [
+		{
+			model: "memberships",
+			opts: { where: [{
+				field: "user_id",
+				value: userId
+			}] }
+		},
+		{
+			model: "account_data",
+			opts: { where: [
+				{
+					field: "user_id",
+					value: userId
+				},
+				{
+					field: "room_id",
+					value: ""
+				},
+				{
+					field: "type",
+					value: M.IGNORED_USER_LIST
+				}
+			] }
+		},
+		{
+			model: "to_device",
+			opts: {
+				where: [...deviceWhere, {
+					field: "stream",
+					operator: "gt",
+					value: p.since?.toDevice ?? 0
+				}],
+				sortBy: {
+					field: "stream",
+					direction: "asc"
+				},
+				limit: 100
+			}
+		},
+		{
+			model: "one_time_keys",
+			opts: {
+				where: [...deviceWhere, {
+					field: "fallback",
+					value: false
+				}],
+				fields: ["algorithm"]
+			}
+		},
+		{
+			model: "fallback_keys",
+			opts: {
+				where: [...deviceWhere, {
+					field: "used",
+					value: false
+				}],
+				fields: ["algorithm"]
+			}
+		},
+		...incremental ? [{
+			model: "account_data",
+			opts: { where: [{
+				field: "user_id",
+				value: userId
+			}, {
+				field: "stream",
+				operator: "gt",
+				value: since
+			}] }
+		}] : []
+	]);
+	p = {
+		...p,
+		ignored: new Set(Object.keys(ignoredRows[0]?.content?.ignored_users ?? {}))
+	};
+	const changes = incremental ? await changesSince(hs, p, memberships, since, accountRows) : null;
+	let memberChangesRead = changes ? Promise.resolve(changes.memberChanges) : null;
+	const memberChanges = () => memberChangesRead ??= hs.rooms.membershipChangesSince(userId, since ?? 0, memberships);
 	const join = {};
 	const invite = {};
 	const leave = {};
 	const knock = {};
 	let empty = true;
+	const joinedBuilds = [];
 	for (const m of memberships) {
 		if (m.stream > p.to || !roomAllowed(p.filter, m.room_id)) continue;
 		const roomId = m.room_id;
 		const changed = since === null || m.stream > since;
 		if (m.membership === "join") {
-			if (!p.filter.room?.state?.lazy_load_members && await hs.partialState.isPartial(roomId)) continue;
-			const completedAt = await hs.partialState.completedAt(roomId);
-			const initial = since === null || p.fullState || m.stream > since || completedAt !== null && completedAt > since && completedAt <= p.to;
-			const { timeline, state, stateAfter, timelineEvents } = await roomTimeline(hs, p, roomId, initial ? null : since, p.to);
-			const ephemeral = await roomEphemeral(hs, p, roomId);
-			const roomAccountData = await accountData(hs, p, roomId);
-			if (!initial && timelineEvents.length === 0 && state.length === 0 && stateAfter.length === 0 && ephemeral.length === 0 && roomAccountData.length === 0) continue;
-			join[roomId] = {
-				timeline,
-				...p.useStateAfter ? { [p.stateAfterKey ?? "state_after"]: { events: stateAfter.map((e) => toClientEvent(e, { includeRoomId: false })) } } : { state: { events: state.map((e) => toClientEvent(e, { includeRoomId: false })) } },
-				ephemeral: { events: ephemeral },
-				account_data: { events: roomAccountData },
-				summary: await roomSummary(hs, roomId, userId)
-			};
-			const counts = await hs.push.unreadCounts(userId, roomId);
-			if (p.filter.room?.timeline?.unread_thread_notifications) {
-				join[roomId].unread_notifications = counts.room;
-				if (Object.keys(counts.threads).length) join[roomId].unread_thread_notifications = counts.threads;
-			} else {
-				const total = { ...counts.room };
-				for (const t of Object.values(counts.threads)) {
-					total.notification_count += t.notification_count;
-					total.highlight_count += t.highlight_count;
-				}
-				join[roomId].unread_notifications = total;
-			}
-			empty = false;
+			if (changes && !changed && !changes.touched.has(roomId) && hs.typing.typingSince(roomId, since) === null) continue;
+			joinedBuilds.push(m);
 		} else if (m.membership === "invite") {
 			if (!changed && !p.fullState) continue;
 			const pdu = await hs.rooms.getPdu(m.event_id);
@@ -341,68 +546,135 @@ async function buildSync(hs, p) {
 			leave[roomId] = {
 				timeline,
 				state: { events: state.map((e) => toClientEvent(e, { includeRoomId: false })) },
-				account_data: { events: await accountData(hs, p, roomId) }
+				account_data: { events: await accountData(hs, p, roomId, changes ? changes.accountData.get(roomId) ?? [] : void 0) }
 			};
 			empty = false;
 		}
 	}
-	const globalAccountData = await accountData(hs, p, "");
+	const joinedRoom = async (m) => {
+		const roomId = m.room_id;
+		if (!p.filter.room?.state?.lazy_load_members && await hs.partialState.isPartial(roomId)) return null;
+		const completedAt = await hs.partialState.completedAt(roomId);
+		const initial = since === null || p.fullState || m.stream > since || completedAt !== null && completedAt > since && completedAt <= p.to;
+		const [{ timeline, state, stateAfter, timelineEvents }, ephemeral, roomAccountData, summary, counts] = await Promise.all([
+			roomTimeline(hs, p, roomId, initial ? null : since, p.to),
+			roomEphemeral(hs, p, roomId, changes ? changes.receipts.get(roomId) ?? [] : void 0),
+			accountData(hs, p, roomId, changes ? changes.accountData.get(roomId) ?? [] : void 0),
+			roomSummary(hs, roomId, userId),
+			hs.push.unreadCounts(userId, roomId)
+		]);
+		if (!initial && timelineEvents.length === 0 && state.length === 0 && stateAfter.length === 0 && ephemeral.length === 0 && roomAccountData.length === 0) return null;
+		const room = {
+			timeline,
+			...p.useStateAfter ? { [p.stateAfterKey ?? "state_after"]: { events: stateAfter.map((e) => toClientEvent(e, { includeRoomId: false })) } } : { state: { events: state.map((e) => toClientEvent(e, { includeRoomId: false })) } },
+			ephemeral: { events: ephemeral },
+			account_data: { events: roomAccountData },
+			summary
+		};
+		if (p.filter.room?.timeline?.unread_thread_notifications) {
+			room.unread_notifications = counts.room;
+			if (Object.keys(counts.threads).length) room.unread_thread_notifications = counts.threads;
+		} else {
+			const total = { ...counts.room };
+			for (const t of Object.values(counts.threads)) {
+				total.notification_count += t.notification_count;
+				total.highlight_count += t.highlight_count;
+			}
+			room.unread_notifications = total;
+		}
+		return room;
+	};
+	const built = await mapConcurrently(joinedBuilds, JOINED_ROOMS_AT_ONCE, joinedRoom);
+	joinedBuilds.forEach((m, i) => {
+		if (!built[i]) return;
+		join[m.room_id] = built[i];
+		empty = false;
+	});
+	const globalAccountData = await accountData(hs, p, "", changes ? changes.accountData.get("") ?? [] : void 0);
 	if (since === null && !globalAccountData.some((e) => e.type === M.PUSH_RULES)) globalAccountData.push({
 		type: M.PUSH_RULES,
-		content: { global: await loadRules(hs, userId) }
+		content: { global: await readOnlyRules(hs, userId) }
 	});
 	if (globalAccountData.length) empty = false;
-	const deviceWhere = [{
-		field: "user_id",
-		value: userId
-	}, {
-		field: "device_id",
-		value: p.session.deviceId
-	}];
-	if (p.since) await hs.db.delete("to_device", [...deviceWhere, {
-		field: "stream",
-		operator: "lte",
-		value: p.since.toDevice
-	}]);
-	const toDevice = (await hs.db.findMany("to_device", {
-		where: [...deviceWhere, {
-			field: "stream",
-			operator: "gt",
-			value: p.since?.toDevice ?? 0
-		}],
-		sortBy: {
-			field: "stream",
-			direction: "asc"
-		},
-		limit: 100
-	})).filter((r) => r.stream <= p.to);
+	await acknowledged;
+	const toDevice = toDeviceRows.filter((r) => r.stream <= p.to);
 	const nextToDevice = toDevice.length ? toDevice[toDevice.length - 1].stream : p.since?.toDevice ?? 0;
 	if (toDevice.length) empty = false;
 	const presenceEvents = [];
-	if (!p.filter.presence?.types || p.filter.presence.types.length) {
-		const sharing = await hs.rooms.usersSharingRooms(userId);
-		const newlyShared = /* @__PURE__ */ new Set();
-		if (since !== null) {
-			const changes = await hs.db.findMany("memberships", { where: [{
+	const wantPresence = !p.filter.presence?.types || p.filter.presence.types.length > 0;
+	const sharing = await hs.rooms.usersSharingRooms(userId);
+	const joinedRooms = memberships.filter((m) => m.membership === "join").map((m) => m.room_id);
+	const newlyShared = /* @__PURE__ */ new Set();
+	if (since !== null && wantPresence) {
+		const myRooms = new Set(joinedRooms);
+		for (const c of await memberChanges()) {
+			if (c.membership !== "join" || !myRooms.has(c.room_id)) continue;
+			if (c.user_id === userId) for (const u of await hs.rooms.joinedMembers(c.room_id)) newlyShared.add(u);
+			else newlyShared.add(c.user_id);
+		}
+	}
+	const sharingList = [...sharing];
+	const recent = (model) => ({
+		model,
+		opts: {
+			where: [{
 				field: "stream",
 				operator: "gt",
 				value: since
-			}] });
-			const myRooms = new Set(memberships.filter((m) => m.membership === "join").map((m) => m.room_id));
-			for (const c of changes) {
-				if (c.membership !== "join" || !myRooms.has(c.room_id)) continue;
-				if (c.user_id === userId) for (const u of await hs.rooms.joinedMembers(c.room_id)) newlyShared.add(u);
-				else newlyShared.add(c.user_id);
-			}
+			}],
+			sortBy: {
+				field: "stream",
+				direction: "asc"
+			},
+			limit: 501
 		}
-		const where = [{
+	});
+	const presenceQuery = since === null ? {
+		model: "presence",
+		opts: { where: [{
+			field: "user_id",
+			operator: "in",
+			value: wantPresence ? sharingList : []
+		}] }
+	} : wantPresence ? recent("presence") : {
+		model: "presence",
+		opts: { where: [{
 			field: "user_id",
 			operator: "in",
-			value: [...sharing]
-		}];
-		for (const row of await hs.db.findMany("presence", { where })) {
+			value: []
+		}] }
+	};
+	let [presenceRows = [], newlySharedPresence = [], deviceRows] = await readBatch(hs.db, [
+		presenceQuery,
+		{
+			model: "presence",
+			opts: { where: [{
+				field: "user_id",
+				operator: "in",
+				value: [...newlyShared]
+			}] }
+		},
+		...since !== null ? [recent("device_list_changes")] : []
+	]);
+	if (since !== null) {
+		if (presenceRows.length > RECENT_CHANGES) presenceRows = await hs.db.findMany("presence", { where: [{
+			field: "user_id",
+			operator: "in",
+			value: sharingList
+		}, {
+			field: "stream",
+			operator: "gt",
+			value: since
+		}] });
+		else presenceRows = presenceRows.filter((r) => sharing.has(r.user_id));
+		if (deviceRows.length > RECENT_CHANGES) deviceRows = await hs.db.findMany("device_list_changes", deviceListChangesQuery(sharing, since));
+	}
+	if (wantPresence) {
+		const seen = /* @__PURE__ */ new Set();
+		for (const row of [...presenceRows, ...newlySharedPresence]) {
+			if (seen.has(row.user_id)) continue;
+			seen.add(row.user_id);
 			if (row.stream > p.to) continue;
-			if (since !== null && row.stream <= since && !newlyShared.has(row.user_id)) continue;
 			const content = {
 				presence: row.presence,
 				last_active_ago: Date.now() - row.last_active_ts
@@ -418,12 +690,13 @@ async function buildSync(hs, p) {
 		}
 	}
 	if (presenceEvents.length) empty = false;
-	const profileUpdates = await profileUpdatesFor(hs, p, memberships);
+	const profileUpdates = await profileUpdatesFor(hs, p, memberships, memberChanges);
 	if (profileUpdates) empty = false;
 	const body = {
 		next_batch: formatToken({
 			pos: p.to,
-			toDevice: nextToDevice
+			toDevice: nextToDevice,
+			ack: toDevice.length > 0
 		}),
 		rooms: Object.fromEntries(Object.entries({
 			join,
@@ -438,12 +711,16 @@ async function buildSync(hs, p) {
 			type: r.type,
 			content: r.content
 		})) },
-		device_one_time_keys_count: await hs.keys.oneTimeKeyCounts(userId, p.session.deviceId),
-		device_unused_fallback_key_types: await hs.keys.unusedFallbackKeyTypes(userId, p.session.deviceId)
+		device_one_time_keys_count: countAlgorithms(oneTimeKeyRows),
+		device_unused_fallback_key_types: [...new Set(fallbackRows.map((r) => r.algorithm))]
 	};
 	if (profileUpdates) body[UNSTABLE.ORG.MATRIX.MSC4429.USERS] = profileUpdates;
 	if (since !== null) {
-		const lists = await hs.keys.changesSince(userId, since, p.to);
+		const lists = await hs.keys.changesSince(userId, since, p.to, {
+			memberChanges: await memberChanges(),
+			deviceRows,
+			joinedRooms
+		});
 		body.device_lists = lists;
 		if (lists.changed.length || lists.left.length) empty = false;
 	}
@@ -453,6 +730,12 @@ async function buildSync(hs, p) {
 		nextToDevice
 	};
 }
+const slots = /* @__PURE__ */ new WeakMap();
+function syncSlots(hs) {
+	let s = slots.get(hs);
+	if (!s) slots.set(hs, s = new Semaphore(hs.config.maxConcurrentSyncs));
+	return s;
+}
 async function sync(ctx) {
 	const hs = ctx.hs;
 	const session = await ctx.auth();
@@ -465,33 +748,21 @@ async function sync(ctx) {
 	const deadline = Date.now() + timeout;
 	for (;;) {
 		const to = await hs.streams.position();
-		const ignoredRow = await hs.db.findOne("account_data", [
-			{
-				field: "user_id",
-				value: session.userId
-			},
-			{
-				field: "room_id",
-				value: ""
-			},
-			{
-				field: "type",
-				value: M.IGNORED_USER_LIST
-			}
-		]);
-		const ignored = new Set(Object.keys(ignoredRow?.content?.ignored_users ?? {}));
+		const ignored = /* @__PURE__ */ new Set();
 		const stable = ctx.query.get("use_state_after") === "true";
 		const unstable = ctx.query.get(UNSTABLE.ORG.MATRIX.MSC4222.USE_STATE_AFTER) === "true";
-		const { body, empty } = await buildSync(hs, {
+		const useStateAfter = stable || unstable;
+		const stateAfterKey = stable ? "state_after" : UNSTABLE.ORG.MATRIX.MSC4222.STATE_AFTER;
+		const { body, empty } = await syncSlots(hs).run(() => buildSync(hs, {
 			session,
 			ignored,
 			since,
 			to,
 			filter,
 			fullState,
-			useStateAfter: stable || unstable,
-			stateAfterKey: stable ? "state_after" : UNSTABLE.ORG.MATRIX.MSC4222.STATE_AFTER
-		});
+			useStateAfter,
+			stateAfterKey
+		}));
 		const remaining = deadline - Date.now();
 		if (!since || !empty || remaining <= 0 || fullState) return body;
 		if (await hs.streams.position() > to) continue;
@@ -499,6 +770,7 @@ async function sync(ctx) {
 		if (ctx.request.signal?.aborted) return body;
 	}
 }
+const LAST_ACTIVE_GRANULARITY_MS = 6e4;
 async function touchPresence(hs, userId, presence) {
 	const state = presence === "unavailable" ? "unavailable" : "online";
 	const row = await hs.db.findOne("presence", [{
@@ -506,7 +778,7 @@ async function touchPresence(hs, userId, presence) {
 		value: userId
 	}]);
 	if (row && row.presence === state) {
-		await hs.db.update("presence", [{
+		if (Date.now() - row.last_active_ts >= LAST_ACTIVE_GRANULARITY_MS) await hs.db.update("presence", [{
 			field: "user_id",
 			value: userId
 		}], { last_active_ts: Date.now() });
@@ -564,7 +836,7 @@ function registerSyncRoutes(r) {
 * MSC4429: profile changes of users sharing a room, limited to the fields the filter asks for.
 * The initial sync carries current values; users we no longer share any room with get `null`.
 */
-async function profileUpdatesFor(hs, p, memberships) {
+async function profileUpdatesFor(hs, p, memberships, memberChanges) {
 	const ids = (p.filter.profile_fields ?? p.filter[UNSTABLE.ORG.MATRIX.MSC4429.PROFILE_FIELDS])?.ids;
 	if (!Array.isArray(ids) || !ids.length) return null;
 	const wanted = new Set(ids.filter((x) => typeof x === "string"));
@@ -577,11 +849,7 @@ async function profileUpdatesFor(hs, p, memberships) {
 	const full = new Set(since === null ? sharing : []);
 	if (since !== null) {
 		const myRooms = new Set(memberships.filter((m) => m.membership === "join").map((m) => m.room_id));
-		const changes = await hs.db.findMany("memberships", { where: [{
-			field: "stream",
-			operator: "gt",
-			value: since
-		}] });
+		const changes = await memberChanges();
 		for (const c of changes) {
 			if (c.stream > p.to || c.user_id === p.session.userId) continue;
 			if (c.membership === "join" && myRooms.has(c.room_id)) {