matrix-nodejs-homeserver 0.3.3 → 0.3.4

package/dist/core/notifier.js

@@ -1,53 +1,150 @@
 //#region src/core/notifier.ts
+/** Users whose latest wake-up position is remembered (#50); the oldest are forgotten first. */
+const RECENT_USERS = 1e5;
+/** How soon held wake-ups are looked at again while the position has not reached them. */
+const RELEASE_RETRY_MS = 10;
 /**
 * Wakes long-polling `/sync` requests when something relevant happens. Notifications
 * are broadcast through the cluster adapter so waiters on other instances wake too.
+*
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/50: every wake-up carries the stream
+* position of its change, or an upper bound of it. Writes can complete out of order, so a change
+* may be committed while an earlier write still holds the position below it. As in Synapse
+* (`Notifier._notify_pending_new_room_events`), such a wake-up is held here until the position
+* reaches it: a request is only woken for changes it can see.
 */
 var Notifier = class {
-	waiters = /* @__PURE__ */ new Map();
+	listeners = /* @__PURE__ */ new Map();
+	/** Each user's latest wake-up position, oldest first (re-inserted on update). */
+	recent = /* @__PURE__ */ new Map();
 	cluster;
 	instanceId;
-	constructor(cluster, instanceId) {
+	lastAllocated;
+	position;
+	/** Wake-ups whose change is beyond the position known to be visible. */
+	held = [];
+	/** The highest position known to be visible. */
+	visible = 0;
+	release = null;
+	/**
+	* `lastAllocated`: the highest stream position this instance has allocated (an upper bound of
+	* its changes); `position`: the position below which everything is visible.
+	*/
+	constructor(cluster, instanceId, lastAllocated = () => 0, position = async () => Number.POSITIVE_INFINITY) {
 		this.cluster = cluster;
 		this.instanceId = instanceId;
+		this.lastAllocated = lastAllocated;
+		this.position = position;
 		cluster.subscribe("notify", (msg) => {
-			if (msg?.from !== this.instanceId && Array.isArray(msg?.users)) this.wakeLocal(msg.users);
+			if (msg?.from !== this.instanceId && Array.isArray(msg?.users)) this.hold(msg.users, Number(msg.pos) || 0);
 		});
 	}
-	notify(userIds) {
+	/**
+	* Wake `userIds`. `pos` is the stream position of the change; without it, the highest position
+	* this instance has allocated stands in, which is never below it.
+	*/
+	notify(userIds, pos) {
 		const users = [...userIds];
 		if (!users.length) return;
-		this.wakeLocal(users);
+		const at = pos ?? this.lastAllocated();
+		this.hold(users, at);
 		this.cluster.publish("notify", {
 			from: this.instanceId,
-			users
+			users,
+			pos: at
 		}).catch(() => {});
 	}
-	wakeLocal(userIds) {
+	/** The position may have moved (a write of this instance completed): release what it covers. */
+	advanced() {
+		if (this.held.length) this.scheduleRelease(0);
+	}
+	/** Wake now if the change is visible, otherwise once the position reaches it. */
+	hold(users, pos) {
+		if (pos <= this.visible) return this.wakeLocal(users, pos);
+		this.held.push({
+			users,
+			pos
+		});
+		this.scheduleRelease(0);
+	}
+	scheduleRelease(delay) {
+		if (this.release) return;
+		this.release = setTimeout(() => {
+			this.release = null;
+			this.releaseHeld().catch(() => this.scheduleRelease(RELEASE_RETRY_MS));
+		}, delay);
+		this.release.unref?.();
+	}
+	async releaseHeld() {
+		const highest = Math.max(...this.held.map((h) => h.pos));
+		let pos = await this.position(false);
+		if (pos < highest) pos = await this.position(true);
+		if (pos > this.visible) this.visible = pos;
+		const due = this.held.filter((h) => h.pos <= this.visible);
+		this.held = this.held.filter((h) => h.pos > this.visible);
+		for (const h of due) this.wakeLocal(h.users, h.pos);
+		if (this.held.length) this.scheduleRelease(RELEASE_RETRY_MS);
+	}
+	wakeLocal(userIds, pos) {
 		for (const u of userIds) {
-			const set = this.waiters.get(u);
+			const previous = this.recent.get(u) ?? 0;
+			this.recent.delete(u);
+			this.recent.set(u, Math.max(previous, pos));
+			if (this.recent.size > RECENT_USERS) this.recent.delete(this.recent.keys().next().value);
+			const set = this.listeners.get(u);
 			if (!set) continue;
-			this.waiters.delete(u);
-			for (const wake of set) wake();
+			this.listeners.delete(u);
+			for (const wake of set) wake(pos);
 		}
 	}
 	notifyAll() {
-		this.wakeLocal([...this.waiters.keys()]);
-	}
-	wait(userId, timeoutMs, signal) {
-		return new Promise((resolve) => {
-			let set = this.waiters.get(userId);
-			if (!set) this.waiters.set(userId, set = /* @__PURE__ */ new Set());
-			const done = () => {
-				clearTimeout(timer);
-				set.delete(done);
-				signal?.removeEventListener("abort", done);
-				resolve();
-			};
-			const timer = setTimeout(done, timeoutMs);
-			signal?.addEventListener("abort", done, { once: true });
-			set.add(done);
-		});
+		for (const [u, set] of this.listeners) {
+			this.listeners.delete(u);
+			for (const wake of set) wake(0);
+		}
+	}
+	listen(userId) {
+		let woken = null;
+		let resolveWait = null;
+		let set = this.listeners.get(userId);
+		if (!set) this.listeners.set(userId, set = /* @__PURE__ */ new Set());
+		const onWake = (pos) => {
+			woken = Math.max(woken ?? 0, pos);
+			resolveWait?.(woken);
+		};
+		set.add(onWake);
+		const remove = () => {
+			const current = this.listeners.get(userId);
+			current?.delete(onWake);
+			if (current && current.size === 0) this.listeners.delete(userId);
+		};
+		return {
+			recent: this.recent.get(userId) ?? 0,
+			wait: (timeoutMs, signal) => {
+				if (woken !== null) return Promise.resolve(woken);
+				return new Promise((resolve) => {
+					const done = (result) => {
+						clearTimeout(timer);
+						signal?.removeEventListener("abort", aborted);
+						resolveWait = null;
+						remove();
+						resolve(result);
+					};
+					const aborted = () => done(null);
+					const timer = setTimeout(() => done(null), timeoutMs);
+					signal?.addEventListener("abort", aborted, { once: true });
+					resolveWait = (pos) => done(pos);
+				});
+			},
+			cancel: () => {
+				remove();
+				resolveWait?.(null);
+			}
+		};
+	}
+	/** Wait for a wake-up of `userId` (for callers that do not track positions). */
+	async wait(userId, timeoutMs, signal) {
+		await this.listen(userId).wait(timeoutMs, signal);
 	}
 };
 //#endregion

package/dist/core/push.d.ts

@@ -7,6 +7,14 @@ import { Homeserver } from "../homeserver.js";
  * Push rule evaluation and push gateway delivery. Every new event is evaluated for
  * every local member; matches are stored for unread counts and `/notifications`.
  */
+type Counts = {
+  notification_count: number;
+  highlight_count: number;
+};
+export interface UnreadCounts {
+  readonly room: Readonly<Counts>;
+  readonly threads: Readonly<Record<string, Readonly<Counts>>>;
+}
 export declare class PushService {
   private hs;
   constructor(hs: Homeserver);
@@ -18,16 +26,8 @@ export declare class PushService {
     unthreaded: number;
     threads: Map<string, number>;
   }): boolean;
-  unreadCounts(userId: string, roomId: string): Promise<{
-    room: {
-      notification_count: number;
-      highlight_count: number;
-    };
-    threads: Record<string, {
-      notification_count: number;
-      highlight_count: number;
-    }>;
-  }>;
+  unreadCounts(userId: string, roomId: string): Promise<UnreadCounts>;
+  private countUnread;
   private deliver;
   /** Rows for `GET /notifications`. */
   list(userId: string, opts: {
@@ -43,4 +43,5 @@ export declare class PushService {
     stream: any;
   }[]>;
 }
-//#endregion
+//#endregion
+export {};

package/dist/core/push.js

@@ -6,6 +6,7 @@ import { globMatch, globSearchWords } from "./glob.js";
 import { KINDS, readOnlyRules } from "./pushrules.js";
 import { isSubscribed } from "../api/threadsubs.js";
 import { upsertMany } from "../storage/batch.js";
+import { freeze } from "./changes.js";
 //#region src/core/push.ts
 /** Look up a dotted path; `\.` escapes a literal dot in a key (e.g. `content.m\.relates_to`). */
 function valueAtPath(obj, path) {
@@ -85,10 +86,6 @@ function threadIdOf(event) {
 	const rel = event.content?.[M.RELATES_TO];
 	return isObject(rel) && rel.rel_type === M.THREAD && typeof rel.event_id === "string" ? rel.event_id : "main";
 }
-/**
-* Push rule evaluation and push gateway delivery. Every new event is evaluated for
-* every local member; matches are stored for unread counts and `/notifications`.
-*/
 var PushService = class {
 	hs;
 	constructor(hs) {
@@ -176,6 +173,11 @@ var PushService = class {
 		return row.stream <= read.unthreaded || row.stream <= (read.threads.get(row.thread_id) ?? 0);
 	}
 	async unreadCounts(userId, roomId) {
+		const changes = this.hs.changes;
+		if (!changes) return this.countUnread(userId, roomId);
+		return changes.values.unreadCounts.get(`${userId}|${roomId}`, async () => freeze(await this.countUnread(userId, roomId)));
+	}
+	async countUnread(userId, roomId) {
 		const read = await this.readPositions(userId, roomId);
 		const unthreaded = read.unthreaded;
 		const rows = await this.hs.db.findMany("notifications", {

package/dist/core/rooms.js

@@ -731,6 +731,12 @@ var RoomService = class {
 			events: touched,
 			sharing: true
 		});
+		const changedMembers = /* @__PURE__ */ new Set();
+		for (const k of /* @__PURE__ */ new Set([...Object.keys(oldIds), ...Object.keys(newIds)])) {
+			const [type, userId] = splitStateKey(k);
+			if (type === M.ROOM.MEMBER && oldIds[k] !== newIds[k] && this.hs.isLocalUser(userId)) changedMembers.add(userId);
+		}
+		if (changedMembers.size) this.hs.notifier.notify([...changedMembers, ...await this.interestedUsers(roomId)]);
 		for (const k of Object.keys(oldIds)) {
 			const [type, userId] = splitStateKey(k);
 			if (type !== M.ROOM.MEMBER || this.hs.isLocalUser(userId) || oldIds[k] === newIds[k]) continue;
@@ -995,7 +1001,7 @@ var RoomService = class {
 				if (this.hs.appservices.registrations.length) await this.hs.appservices.onEvent(view, newState).catch((err) => this.hs.logger.warn("Appservice push failed", err));
 				if (isState) await this.hs.delayedEvents.onStateEvent(roomId, pdu.type, pdu.state_key, sender).catch(() => {});
 				wake = async () => {
-					this.hs.notifier.notify(await this.interestedUsers(roomId));
+					this.hs.notifier.notify(await this.interestedUsers(roomId), backfill ? void 0 : stream);
 					this.hs.emit("event", toClientEvent(view));
 					if (this.hs.adminRoom) this.hs.adminRoom.onEvent(view).catch((err) => this.hs.logger.error("Admin command failed", err));
 				};

package/dist/core/semaphore.js

@@ -14,7 +14,7 @@ var Semaphore = class {
 			return await fn();
 		} finally {
 			const next = this.waiting.shift();
-			if (next) next();
+			if (next) setImmediate(next);
 			else this.active--;
 		}
 	}

package/dist/homeserver.d.ts

@@ -26,6 +26,7 @@ import { SettingsService } from "./core/settings.js";
 import { AdminRoomOptions, AdminRoomService } from "./core/adminroom.js";
 import { ResolvedUserAuth, UserAuthAdapter } from "./adapters/auth.js";
 import { MediaAdapter, Thumbnailer } from "./adapters/media.js";
+import { ChangeTracker } from "./core/changes.js";
 import { AddressPolicy } from "./core/outbound.js";
 import { EventEmitter } from "node:events";
 //#region src/homeserver.d.ts
@@ -84,7 +85,7 @@ export interface HomeserverOptions {
   /** Maximum `/sync` long-poll duration in ms. Default: 30000. */
   maxSyncTimeoutMs?: number;
   /**
-   * How many `/sync` responses one process builds at once (default 4). Requests woken together
+   * How many `/sync` responses one process builds at once (default 16). Requests woken together
    * wait in the process instead of queueing their database reads ahead of everything else.
    */
   maxConcurrentSyncs?: number;
@@ -176,6 +177,11 @@ export declare class Homeserver extends EventEmitter {
   readonly serverName: string;
   readonly config: ResolvedConfig;
   readonly db: DatabaseAdapter;
+  /**
+   * Where keys last changed, when this is the only instance on its database (no cluster
+   * adapter): /sync skips the reads that would find nothing new (#51). Null in a cluster.
+   */
+  readonly changes: ChangeTracker | null;
   readonly media: MediaAdapter;
   readonly userAuth: ResolvedUserAuth;
   readonly logger: Logger;

package/dist/homeserver.js

@@ -14,6 +14,7 @@ import { Notifier } from "./core/notifier.js";
 import { StreamManager, localCluster, newInstanceId } from "./core/cluster.js";
 import { TypingService } from "./core/typing.js";
 import { coalesceReads } from "./storage/batch.js";
+import { ChangeTracker } from "./core/changes.js";
 import { PushService } from "./core/push.js";
 import { SsoService } from "./core/sso.js";
 import { AdminService } from "./core/admin.js";
@@ -53,6 +54,11 @@ var Homeserver = class extends EventEmitter {
 	serverName;
 	config;
 	db;
+	/**
+	* Where keys last changed, when this is the only instance on its database (no cluster
+	* adapter): /sync skips the reads that would find nothing new (#51). Null in a cluster.
+	*/
+	changes;
 	media;
 	userAuth;
 	logger;
@@ -115,7 +121,7 @@ var Homeserver = class extends EventEmitter {
 			defaultRoomVersion,
 			accessTokenLifetimeMs: options.accessTokenLifetimeMs ?? null,
 			maxSyncTimeoutMs: options.maxSyncTimeoutMs ?? 3e4,
-			maxConcurrentSyncs: Math.max(1, options.maxConcurrentSyncs ?? 4),
+			maxConcurrentSyncs: Math.max(1, options.maxConcurrentSyncs ?? 16),
 			urlPreviews: options.urlPreviews ? { allowPrivateAddresses: typeof options.urlPreviews === "object" && !!options.urlPreviews.allowPrivateAddresses } : null,
 			thumbnailer: options.thumbnailer ?? null,
 			rtc: options.rtc ?? {},
@@ -125,19 +131,21 @@ var Homeserver = class extends EventEmitter {
 				roomName: options.serverNotices?.roomName ?? "Server Notices"
 			}
 		};
-		const db = options.databaseAdapter;
-		if (db && typeof db.findOne === "function") this.db = coalesceReads(db);
-		else {
+		this.changes = options.cluster ? null : new ChangeTracker();
+		let db = options.databaseAdapter;
+		if (!db || typeof db.findOne !== "function") {
 			this.logger.warn("No database adapter given; using in-memory storage (data is lost on restart)");
-			this.db = memoryAdapter();
+			db = memoryAdapter();
 		}
+		this.db = coalesceReads(this.changes ? this.changes.wrap(db) : db);
 		const media = options.mediaAdapter;
 		this.media = media && typeof media.put === "function" ? media : memoryMedia();
 		this.userAuth = resolveUserAuth(options.userAuthAdapter);
 		this.cluster = options.cluster ?? localCluster();
 		this.instanceId = newInstanceId();
-		this.streams = new StreamManager(this.cluster, this.instanceId);
-		this.notifier = new Notifier(this.cluster, this.instanceId);
+		this.streams = new StreamManager(this.cluster, this.instanceId, (...args) => this.logger.warn(...args));
+		this.notifier = new Notifier(this.cluster, this.instanceId, () => this.streams.lastAllocated, (fresh) => this.streams.position(fresh));
+		this.streams.onAdvance = () => this.notifier.advanced();
 		this.auth = new AuthService(this);
 		this.rooms = new RoomService(this, !options.cluster);
 		this.keys = new KeyService(this);
@@ -197,6 +205,7 @@ var Homeserver = class extends EventEmitter {
 		}
 		const snap = await this.db.max("state_snapshots", "id");
 		if (snap !== null) await this.streams.advanceTo(snap);
+		this.changes?.start(await this.streams.position(true));
 		const oldest = await this.db.findMany("events", {
 			sortBy: {
 				field: "stream",

package/dist/storage/sql.js

@@ -207,8 +207,8 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 	};
 	return build(rootExecutor, false);
 }
-/** How long SQLite statements may run back to back before yielding to the event loop (#47). */
-const YIELD_AFTER_MS = 10;
+/** How long SQLite statements may run back to back before yielding to the event loop (#47, #52). */
+const YIELD_AFTER_MS = 50;
 /** Prepared statements an SQLite adapter keeps (#40). */
 const STATEMENT_CACHE = 500;
 /** SQLite adapter for `node:sqlite` (`new DatabaseSync(path)`) or `better-sqlite3`. */

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-nodejs-homeserver",
-  "version": "0.3.3",
+  "version": "0.3.4",
   "description": "A framework- and database-agnostic Matrix homeserver library for Node.js",
   "keywords": [
     "chat",