matrix-nodejs-homeserver 0.3.1 → 0.3.3

package/dist/core/pushrules.js

@@ -131,14 +131,33 @@ function cacheFor(hs) {
 	}
 	return c;
 }
+/** A user's push rules, as a copy the caller may change. */
 async function loadRules(hs, userId) {
+	return structuredClone(await readOnlyRules(hs, userId));
+}
+/**
+* A user's push rules, shared and frozen: for callers that only read them.
+*
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/41: push evaluation copied every
+* member's rules for every message, about 6% of the CPU under load, though it only reads them.
+* The cached rulesets are deep-frozen, so a caller that tries to change one fails loudly instead
+* of changing everyone's.
+*/
+async function readOnlyRules(hs, userId) {
 	const cached = cacheFor(hs).get(userId);
-	if (cached) return structuredClone(cached);
-	const rules = await readRules(hs, userId);
+	if (cached) return cached;
+	const rules = deepFreeze(await readRules(hs, userId));
 	const cache = cacheFor(hs);
 	if (cache.size > 5e3) cache.clear();
 	cache.set(userId, rules);
-	return structuredClone(rules);
+	return rules;
+}
+function deepFreeze(value) {
+	if (value && typeof value === "object" && !Object.isFrozen(value)) {
+		Object.freeze(value);
+		for (const v of Object.values(value)) deepFreeze(v);
+	}
+	return value;
 }
 async function readRules(hs, userId) {
 	const row = await hs.db.findOne("account_data", [
@@ -162,7 +181,7 @@ async function readRules(hs, userId) {
 	return rules;
 }
 async function saveRules(hs, userId, rules) {
-	cacheFor(hs).set(userId, structuredClone(rules));
+	cacheFor(hs).set(userId, deepFreeze(structuredClone(rules)));
 	hs.cluster.publish("pushrules", {
 		from: hs.instanceId,
 		userId
@@ -226,4 +245,4 @@ async function copyRules(hs, userId, oldRoomId, newRoomId) {
 	if (changed) await saveRules(hs, userId, rules);
 }
 //#endregion
-export { KINDS, copyRoomPushRules, defaultRules, loadRules, saveRules, withRulesLock };
+export { KINDS, copyRoomPushRules, defaultRules, loadRules, readOnlyRules, saveRules, withRulesLock };

package/dist/core/ratelimit.d.ts

@@ -1,5 +1,5 @@
 //#region src/core/ratelimit.d.ts
-/** A rate limiter keyed by an arbitrary string. Implement this to share limits across instances (e.g. Redis). */
+/** A rate limiter keyed by an arbitrary string. Implement this to share limits across instances. */
 export interface RateLimiter {
   /** Consume one unit for `key` under `rule`; resolve with the delay to wait (0 when allowed). */
   consume(key: string, rule: RateLimitRule): Promise<number>;

package/dist/core/rooms.d.ts

@@ -1,7 +1,7 @@
 import { ClientEvent, PDU } from "./events.js";
 import { StateMap } from "./authrules.js";
-import { StateIds } from "./stateres.js";
 import { Row } from "../storage/types.js";
+import { StateIds } from "./stateres.js";
 import { Homeserver } from "../homeserver.js";
 //#region src/core/rooms.d.ts
 export interface StoredEvent {
@@ -140,6 +140,15 @@ export declare class RoomService {
   /** Users that should be woken up for events in this room. */
   private interestedUsers;
   roomsForUser(userId: string, membership?: string): Promise<string[]>;
+  /**
+   * Membership changes after `since` in the rooms where `userId` has a membership (in any state),
+   * given the user's own membership rows if the caller has them.
+   *
+   * https://github.com/k18k/matrix-nodejs-homeserver/issues/31: /sync and device-list tracking
+   * read every membership change on the server since their token and kept those in the user's
+   * rooms; while many users joined, each sync parsed thousands of rows. This reads those rooms only.
+   */
+  membershipChangesSince(userId: string, since: number, mine?: Row[]): Promise<Row[]>;
   /** All users sharing at least one joined room with `userId` (including themselves). */
   usersSharingRooms(userId: string): Promise<Set<string>>;
   private prepareRedaction;

package/dist/core/rooms.js

@@ -1053,6 +1053,31 @@ var RoomService = class {
 			value: membership
 		}] })).map((r) => r.room_id);
 	}
+	/**
+	* Membership changes after `since` in the rooms where `userId` has a membership (in any state),
+	* given the user's own membership rows if the caller has them.
+	*
+	* https://github.com/k18k/matrix-nodejs-homeserver/issues/31: /sync and device-list tracking
+	* read every membership change on the server since their token and kept those in the user's
+	* rooms; while many users joined, each sync parsed thousands of rows. This reads those rooms only.
+	*/
+	async membershipChangesSince(userId, since, mine) {
+		const own = mine ?? await this.hs.db.findMany("memberships", { where: [{
+			field: "user_id",
+			value: userId
+		}] });
+		const rooms = [...new Set(own.map((m) => m.room_id))];
+		if (rooms.length === 0) return [];
+		return this.hs.db.findMany("memberships", { where: [{
+			field: "room_id",
+			operator: "in",
+			value: rooms
+		}, {
+			field: "stream",
+			operator: "gt",
+			value: since
+		}] });
+	}
 	/** All users sharing at least one joined room with `userId` (including themselves). */
 	async usersSharingRooms(userId) {
 		const cached = this.sharingCache.get(userId);

package/dist/core/semaphore.js

@@ -0,0 +1,23 @@
+//#region src/core/semaphore.ts
+/** Runs at most `limit` tasks at once; the others wait their turn, first come first served. */
+var Semaphore = class {
+	limit;
+	active = 0;
+	waiting = [];
+	constructor(limit) {
+		this.limit = limit;
+	}
+	async run(fn) {
+		if (this.active >= this.limit) await new Promise((resolve) => this.waiting.push(resolve));
+		else this.active++;
+		try {
+			return await fn();
+		} finally {
+			const next = this.waiting.shift();
+			if (next) next();
+			else this.active--;
+		}
+	}
+};
+//#endregion
+export { Semaphore };

package/dist/core/sso.d.ts

@@ -28,11 +28,25 @@ export interface SsoProvider {
 }
 export interface SsoOptions {
   providers: SsoProvider[];
+  /**
+   * Clients that receive the login token straight away, as URL prefixes such as
+   * `"https://app.element.io/"` (same scheme and host, path starting with the prefix's path).
+   * Other clients get a page naming them, where the user confirms before the token is sent.
+   */
+  clientAllowlist?: string[];
+}
+/** Where the browser goes after login, and whether the user has to confirm that client first. */
+export interface SsoResult {
+  url: string;
+  confirm: boolean;
+  /** The client as shown to the user: its host, or its scheme for app links. */
+  client: string;
 }
 /** Upstream OIDC login (authorization code + PKCE) issuing Matrix login tokens. */
 export declare class SsoService {
   private hs;
   readonly providers: SsoProvider[];
+  private clientAllowlist;
   private discovery;
   constructor(hs: Homeserver, options: SsoOptions);
   private callbackUrl;
@@ -40,7 +54,7 @@ export declare class SsoService {
   /** URL to send the user's browser to. */
   startLogin(providerId: string | null, redirectUrl: string): Promise<string>;
   /** Handle the provider's redirect; returns the client URL carrying a login token. */
-  finishLogin(providerId: string, params: URLSearchParams): Promise<string>;
+  finishLogin(providerId: string, params: URLSearchParams): Promise<SsoResult>;
   private resolveUser;
 }
 //#endregion

package/dist/core/sso.js

@@ -3,6 +3,34 @@ import { errors } from "./errors.js";
 import { LOCALPART_RE } from "./ids.js";
 import { createHash, randomBytes } from "node:crypto";
 //#region src/core/sso.ts
+const UNSAFE_SCHEMES = /* @__PURE__ */ new Set([
+	"javascript:",
+	"data:",
+	"vbscript:",
+	"file:",
+	"blob:",
+	"about:"
+]);
+/** The client's redirect URL, if it is one the server may send a login token to. */
+function checkRedirectUrl(raw) {
+	if (!raw) return null;
+	try {
+		return UNSAFE_SCHEMES.has(new URL(raw).protocol) ? null : raw;
+	} catch {
+		return null;
+	}
+}
+function clientMatches(raw, entry) {
+	try {
+		const url = new URL(raw);
+		const allowed = new URL(entry);
+		if (allowed.host) return url.protocol === allowed.protocol && url.host === allowed.host && url.pathname.startsWith(allowed.pathname);
+		return raw.startsWith(entry);
+	} catch {
+		return false;
+	}
+}
+const sameIssuer = (a, b) => typeof a === "string" && a.replace(/\/$/, "") === b.replace(/\/$/, "");
 function sanitizeLocalpart(raw) {
 	return raw.toLowerCase().replace(/[^a-z0-9._=\-/+]/g, "_").slice(0, 200);
 }
@@ -15,10 +43,12 @@ function decodeJwtPayload(jwt) {
 var SsoService = class {
 	hs;
 	providers;
+	clientAllowlist;
 	discovery = /* @__PURE__ */ new Map();
 	constructor(hs, options) {
 		this.hs = hs;
 		this.providers = options.providers;
+		this.clientAllowlist = options.clientAllowlist ?? [];
 	}
 	callbackUrl(provider) {
 		const base = this.hs.config.publicBaseUrl;
@@ -31,6 +61,7 @@ var SsoService = class {
 		const res = await this.hs.fetch(`${provider.issuer.replace(/\/$/, "")}/.well-known/openid-configuration`, { signal: AbortSignal.timeout(1e4) });
 		if (!res.ok) throw new Error(`OIDC discovery for ${provider.id} failed with ${res.status}`);
 		const meta = await res.json();
+		if (!sameIssuer(meta.issuer, provider.issuer)) throw new Error(`OIDC discovery for ${provider.id} returned another issuer`);
 		this.discovery.set(provider.id, meta);
 		return meta;
 	}
@@ -96,6 +127,7 @@ var SsoService = class {
 		if (!res.ok) throw errors.forbidden("Code exchange failed");
 		const tokens = await res.json();
 		let claims = {};
+		if ((provider.scopes ?? ["openid"]).includes("openid") && typeof tokens.id_token !== "string") throw errors.forbidden("The provider returned no ID token");
 		if (typeof tokens.id_token === "string") {
 			claims = decodeJwtPayload(tokens.id_token);
 			const aud = Array.isArray(claims.aud) ? claims.aud : [claims.aud];
@@ -130,7 +162,11 @@ var SsoService = class {
 		});
 		const redirect = new URL(pending.redirectUrl);
 		redirect.searchParams.set("loginToken", loginToken);
-		return redirect.toString();
+		return {
+			url: redirect.toString(),
+			confirm: !this.clientAllowlist.some((entry) => clientMatches(pending.redirectUrl, entry)),
+			client: redirect.host || redirect.protocol.replace(/:$/, "")
+		};
 	}
 	async resolveUser(provider, claims) {
 		const link = await this.hs.db.findOne("sso_identities", [{
@@ -162,4 +198,4 @@ var SsoService = class {
 	}
 };
 //#endregion
-export { SsoService };
+export { SsoService, checkRedirectUrl };

package/dist/core/threepid.js

@@ -1,6 +1,7 @@
 import { randomString } from "./crypto.js";
 import { MatrixError, errors } from "./errors.js";
 import { toMsisdn } from "./phone.js";
+import { escapeHtml } from "./html.js";
 import { randomInt } from "node:crypto";
 //#region src/core/threepid.ts
 const CLIENT_SECRET_RE = /^[0-9a-zA-Z.=_-]{1,255}$/;
@@ -262,14 +263,5 @@ var ThreepidService = class {
 		};
 	}
 };
-function escapeHtml(s) {
-	return s.replace(/[&<>"']/g, (c) => ({
-		"&": "&amp;",
-		"<": "&lt;",
-		">": "&gt;",
-		"\"": "&quot;",
-		"'": "&#39;"
-	})[c]);
-}
 //#endregion
 export { ThreepidService, canonicalAddress, isMedium };

package/dist/federation/client.js

@@ -2,6 +2,7 @@ import { canonicalJson } from "../core/json.js";
 import { unpaddedBase64 } from "../core/crypto.js";
 import { M } from "../core/m.js";
 import { MatrixError } from "../core/errors.js";
+import { checkLiteral } from "../core/outbound.js";
 import { sign } from "node:crypto";
 import { resolveSrv } from "node:dns/promises";
 import { Agent, request } from "node:https";
@@ -169,6 +170,11 @@ var FederationClient = class {
 		return null;
 	}
 	raw(target, method, pathWithQuery, body, headers, timeoutMs, poolKey = target.hostHeader) {
+		try {
+			checkLiteral(this.hs.addressPolicy, target.host);
+		} catch (err) {
+			return Promise.reject(err);
+		}
 		return new Promise((resolve, reject) => {
 			let connectTimer = null;
 			const req = request({
@@ -178,6 +184,7 @@ var FederationClient = class {
 				path: pathWithQuery,
 				servername: isIP(target.tlsName) ? void 0 : target.tlsName,
 				agent: this.agentFor(poolKey),
+				lookup: this.hs.addressPolicy.lookup,
 				headers: {
 					host: target.hostHeader,
 					...headers,

package/dist/federation/index.d.ts

@@ -136,7 +136,7 @@ export declare class FederationService {
    */
   backfill(roomId: string, fromEventIds: string[], limit?: number): Promise<number>;
   /** Download remote media (authenticated federation endpoint, falling back to the legacy one). */
-  fetchMedia(server: string, mediaId: string): Promise<{
+  fetchMedia(server: string, mediaId: string, maxBytes: number): Promise<{
     data: Uint8Array;
     contentType: string | null;
     filename: string | null;

package/dist/federation/index.js

@@ -8,6 +8,7 @@ import { authStateFromAuthEvents, checkAuth, getState, membershipOf } from "../c
 import { globMatch } from "../core/glob.js";
 import { resolveState } from "../core/stateres.js";
 import { MissingStateError } from "../core/rooms.js";
+import { publicGet } from "../core/outbound.js";
 import { FederationClient } from "./client.js";
 import { KeyRing } from "./keyring.js";
 import { TransactionSender } from "./sender.js";
@@ -961,13 +962,16 @@ var FederationService = class {
 		return 0;
 	}
 	/** Download remote media (authenticated federation endpoint, falling back to the legacy one). */
-	async fetchMedia(server, mediaId) {
+	async fetchMedia(server, mediaId, maxBytes) {
 		this.requireEnabled();
 		const res = await this.client.requestRaw(server, "GET", `/_matrix/federation/v1/media/download/${encodeURIComponent(mediaId)}`, { timeoutMs: 6e4 });
 		if (res.status === 200) {
 			const ct = String(res.headers["content-type"] ?? "");
 			const m = /boundary=("?)([^";]+)\1/.exec(ct);
-			if (ct.startsWith("multipart/") && m) return parseMultipartMedia(res.body, m[2], this);
+			if (ct.startsWith("multipart/") && m) return parseMultipartMedia(res.body, m[2], {
+				maxBytes,
+				fetch: this.hs.publicFetch
+			});
 		}
 		const legacy = await this.client.requestRaw(server, "GET", `/_matrix/media/v3/download/${encodeURIComponent(server)}/${encodeURIComponent(mediaId)}`, {
 			query: { allow_remote: "false" },
@@ -1020,7 +1024,7 @@ function filenameFromDisposition(header) {
 	return plain ? plain[1] : null;
 }
 /** Parse a federation media multipart/mixed response: a JSON part then the file (or a redirect). */
-async function parseMultipartMedia(body, boundary, fed) {
+async function parseMultipartMedia(body, boundary, limits) {
 	const delimiter = Buffer.from(`--${boundary}`);
 	const parts = [];
 	let idx = body.indexOf(delimiter);
@@ -1041,10 +1045,13 @@ async function parseMultipartMedia(body, boundary, fed) {
 		if (i > 0) headers[line.slice(0, i).trim().toLowerCase()] = line.slice(i + 1).trim();
 	}
 	if (headers.location) {
-		const res = await fetch(headers.location, { signal: AbortSignal.timeout(6e4) });
-		if (!res.ok) throw errors.notFound("Remote media redirect failed");
+		const res = await publicGet(limits.fetch, headers.location, {
+			maxBytes: limits.maxBytes,
+			timeoutMs: 6e4
+		});
+		if (res.status !== 200) throw errors.notFound("Remote media redirect failed");
 		return {
-			data: new Uint8Array(await res.arrayBuffer()),
+			data: new Uint8Array(res.body),
 			contentType: res.headers.get("content-type"),
 			filename: filenameFromDisposition(res.headers.get("content-disposition") ?? void 0)
 		};

package/dist/homeserver.d.ts

@@ -6,10 +6,10 @@ import { IdentityServers } from "./core/identity.js";
 import { M } from "./core/m.js";
 import { AppServiceRegistration, AppServiceService } from "./core/appservices.js";
 import { AuthService } from "./core/auth.js";
+import { DatabaseAdapter } from "./storage/types.js";
 import { KeyService } from "./core/keys.js";
 import { ClusterAdapter, StreamManager } from "./core/cluster.js";
 import { Notifier } from "./core/notifier.js";
-import { DatabaseAdapter } from "./storage/types.js";
 import { RoomService } from "./core/rooms.js";
 import { TypingService } from "./core/typing.js";
 import { PushService } from "./core/push.js";
@@ -26,6 +26,7 @@ import { SettingsService } from "./core/settings.js";
 import { AdminRoomOptions, AdminRoomService } from "./core/adminroom.js";
 import { ResolvedUserAuth, UserAuthAdapter } from "./adapters/auth.js";
 import { MediaAdapter, Thumbnailer } from "./adapters/media.js";
+import { AddressPolicy } from "./core/outbound.js";
 import { EventEmitter } from "node:events";
 //#region src/homeserver.d.ts
 export interface Logger {
@@ -82,6 +83,11 @@ export interface HomeserverOptions {
   accessTokenLifetimeMs?: number;
   /** Maximum `/sync` long-poll duration in ms. Default: 30000. */
   maxSyncTimeoutMs?: number;
+  /**
+   * How many `/sync` responses one process builds at once (default 4). Requests woken together
+   * wait in the process instead of queueing their database reads ahead of everything else.
+   */
+  maxConcurrentSyncs?: number;
   /**
    * Enables `/preview_url`. Off by default because it makes the server fetch arbitrary URLs.
    * Private/internal addresses stay blocked unless `allowPrivateAddresses` is set.
@@ -92,6 +98,13 @@ export interface HomeserverOptions {
   logger?: Logger;
   /** Custom `fetch` for outgoing HTTP requests. */
   fetch?: typeof globalThis.fetch;
+  /**
+   * Let requests to addresses that users or remote servers choose (federation, push gateways,
+   * identity servers, URL previews, remote media) reach private addresses: localhost, LAN and
+   * other non-public ranges. `true` allows all of them, a list allows those ranges only (e.g.
+   * `["127.0.0.1/32"]` for a push gateway on the same machine). Default: none.
+   */
+  allowPrivateAddresses?: boolean | string[];
   /**
    * Rate limiting. `false` disables it; otherwise per-action overrides and an optional
    * shared limiter (defaults to an in-memory token bucket). Admins are never limited.
@@ -102,7 +115,7 @@ export interface HomeserverOptions {
   sso?: SsoOptions;
   /**
    * Coordination between instances. The default (`localCluster()`) supports one process;
-   * use `postgresCluster()` / `redisCluster()` to scale horizontally.
+   * use `postgresCluster()` to scale horizontally.
    */
   cluster?: ClusterAdapter;
   /**
@@ -151,6 +164,7 @@ export interface ResolvedConfig {
   defaultRoomVersion: string;
   accessTokenLifetimeMs: number | null;
   maxSyncTimeoutMs: number;
+  maxConcurrentSyncs: number;
   urlPreviews: {
     allowPrivateAddresses: boolean;
   } | null;
@@ -189,8 +203,12 @@ export declare class Homeserver extends EventEmitter {
   readonly policyServers: PolicyServers;
   readonly threepids: ThreepidService;
   readonly identityServers: IdentityServers;
-  /** `fetch` used for outgoing requests (push gateways, URL previews, OAuth). Overridable for testing. */
+  /** `fetch` for requests to addresses the operator configured (OAuth, SSO, application services). */
   readonly fetch: typeof globalThis.fetch;
+  /** Which addresses requests to user- or remote-chosen destinations may reach (see `allowPrivateAddresses`). */
+  readonly addressPolicy: AddressPolicy;
+  /** `fetch` for user- or remote-chosen URLs: refuses private addresses unless allowed, never follows redirects. */
+  readonly publicFetch: typeof globalThis.fetch;
   /** Resolves once storage is migrated and the server is ready to serve requests. */
   readonly ready: Promise<void>;
   private _signingKey;

package/dist/homeserver.js

@@ -3,6 +3,7 @@ import "./core/m.js";
 import { SUPPORTED_ROOM_VERSIONS } from "./core/events.js";
 import { PartialStateService } from "./core/partialstate.js";
 import { RoomService } from "./core/rooms.js";
+import { addressPolicy, policyFetch } from "./core/outbound.js";
 import { FederationService } from "./federation/index.js";
 import { PolicyServers } from "./core/policy.js";
 import { ThreepidService } from "./core/threepid.js";
@@ -12,10 +13,11 @@ import { KeyService } from "./core/keys.js";
 import { Notifier } from "./core/notifier.js";
 import { StreamManager, localCluster, newInstanceId } from "./core/cluster.js";
 import { TypingService } from "./core/typing.js";
+import { coalesceReads } from "./storage/batch.js";
 import { PushService } from "./core/push.js";
+import { SsoService } from "./core/sso.js";
 import { AdminService } from "./core/admin.js";
 import { OAuthDelegation } from "./core/oauth.js";
-import { SsoService } from "./core/sso.js";
 import { DelayedEventService } from "./core/delayed.js";
 import { AuditService } from "./core/audit.js";
 import { SettingsService } from "./core/settings.js";
@@ -78,8 +80,12 @@ var Homeserver = class extends EventEmitter {
 	policyServers;
 	threepids;
 	identityServers;
-	/** `fetch` used for outgoing requests (push gateways, URL previews, OAuth). Overridable for testing. */
+	/** `fetch` for requests to addresses the operator configured (OAuth, SSO, application services). */
 	fetch;
+	/** Which addresses requests to user- or remote-chosen destinations may reach (see `allowPrivateAddresses`). */
+	addressPolicy;
+	/** `fetch` for user- or remote-chosen URLs: refuses private addresses unless allowed, never follows redirects. */
+	publicFetch;
 	/** Resolves once storage is migrated and the server is ready to serve requests. */
 	ready;
 	_signingKey = null;
@@ -109,6 +115,7 @@ var Homeserver = class extends EventEmitter {
 			defaultRoomVersion,
 			accessTokenLifetimeMs: options.accessTokenLifetimeMs ?? null,
 			maxSyncTimeoutMs: options.maxSyncTimeoutMs ?? 3e4,
+			maxConcurrentSyncs: Math.max(1, options.maxConcurrentSyncs ?? 4),
 			urlPreviews: options.urlPreviews ? { allowPrivateAddresses: typeof options.urlPreviews === "object" && !!options.urlPreviews.allowPrivateAddresses } : null,
 			thumbnailer: options.thumbnailer ?? null,
 			rtc: options.rtc ?? {},
@@ -119,7 +126,7 @@ var Homeserver = class extends EventEmitter {
 			}
 		};
 		const db = options.databaseAdapter;
-		if (db && typeof db.findOne === "function") this.db = db;
+		if (db && typeof db.findOne === "function") this.db = coalesceReads(db);
 		else {
 			this.logger.warn("No database adapter given; using in-memory storage (data is lost on restart)");
 			this.db = memoryAdapter();
@@ -138,6 +145,9 @@ var Homeserver = class extends EventEmitter {
 		this.push = new PushService(this);
 		this.admin = new AdminService(this);
 		this.oauth = options.oauth ? new OAuthDelegation(this, options.oauth) : null;
+		this.fetch = options.fetch ?? globalThis.fetch;
+		this.addressPolicy = addressPolicy(options.allowPrivateAddresses);
+		this.publicFetch = policyFetch(this.addressPolicy, options.fetch);
 		this.federation = new FederationService(this, options.federation);
 		this.delayedEvents = new DelayedEventService(this, options.delayedEvents);
 		this.audit = new AuditService(this, options.audit);
@@ -157,7 +167,6 @@ var Homeserver = class extends EventEmitter {
 			this.cleanupTimer = setInterval(() => void this.cleanup().catch((err) => this.logger.warn("Cleanup failed", err)), cleanupEvery);
 			this.cleanupTimer.unref?.();
 		}
-		this.fetch = options.fetch ?? globalThis.fetch;
 		registerRoutes(this.router);
 		this.ready = this.init(options);
 		this.ready.catch((err) => this.logger.error("Homeserver failed to start", err));

package/dist/index.d.ts

@@ -1,8 +1,8 @@
 import { ThreepidEmail, ThreepidOptions, ThreepidPurpose, ThreepidSms } from "./core/threepid.js";
 import { M, UNSTABLE } from "./core/m.js";
 import { AppServiceRegistration, loadAppServiceRegistrations, parseAppServiceRegistration } from "./core/appservices.js";
-import { ClusterAdapter, localCluster } from "./core/cluster.js";
 import { DatabaseAdapter, FieldDef, FindManyOptions, ModelDef, Operator, Row, Schema, WhereClause } from "./storage/types.js";
+import { ClusterAdapter, localCluster } from "./core/cluster.js";
 import { AdminService, ServerNoticesOptions } from "./core/admin.js";
 import { OAuthDelegationOptions } from "./core/oauth.js";
 import { SsoOptions, SsoProvider } from "./core/sso.js";
@@ -22,5 +22,5 @@ import { memoryAdapter } from "./storage/memory.js";
 import { MysqlPoolLike, PgPoolLike, SqlAdapterOptions, SqlDialect, SqlExecutor, SqliteDatabaseLike, mysqlAdapter, postgresAdapter, sqlAdapter, sqliteAdapter } from "./storage/sql.js";
 import { schema } from "./storage/schema.js";
 import { S3MediaOptions, s3Media } from "./adapters/s3.js";
-import { PgClusterPoolLike, RedisLike, RedisSubscriberLike, postgresCluster, redisCluster } from "./adapters/cluster.js";
-export { type AdminRoomOptions, type AdminService, type AppServiceRegistration, type AuditAction, type AuditEntry, type AuditOptions, type ClusterAdapter, type DatabaseAdapter, type DelayedEventsOptions, type FederationOptions, type FieldDef, type FindManyOptions, Homeserver, type HomeserverOptions, type Logger, M, MatrixError, type MediaAdapter, type MediaMetadata, type ModelDef, type MysqlPoolLike, type NodeHandler, type OAuthDelegationOptions, type Operator, type PasswordCheckContext, type PgClusterPoolLike, type PgPoolLike, type RateLimitAction, type RateLimitRule, type RateLimiter, type RedisLike, type RedisSubscriberLike, type Row, type RtcOptions, type S3MediaOptions, type Schema, type ServerNoticesOptions, type SqlAdapterOptions, type SqlDialect, type SqlExecutor, type SqliteDatabaseLike, type SsoOptions, type SsoProvider, type ThreepidEmail, type ThreepidOptions, type ThreepidPurpose, type ThreepidSms, type ThumbnailRequest, type Thumbnailer, type TurnOptions, UNSTABLE, type UserAuthAdapter, type WhereClause, createHomeserver, filesystemMedia, getExpressHandler, getFastifyPlugin, getFetchHandler, getHonoHandler, getNodeHandler, handleRequest, isMatrixPath, loadAppServiceRegistrations, localCluster, memoryAdapter, memoryMedia, memoryRateLimiter, mysqlAdapter, parseAppServiceRegistration, postgresAdapter, postgresCluster, redisCluster, s3Media, schema, sharpThumbnailer, sqlAdapter, sqliteAdapter };
+import { PgClusterPoolLike, postgresCluster } from "./adapters/cluster.js";
+export { type AdminRoomOptions, type AdminService, type AppServiceRegistration, type AuditAction, type AuditEntry, type AuditOptions, type ClusterAdapter, type DatabaseAdapter, type DelayedEventsOptions, type FederationOptions, type FieldDef, type FindManyOptions, Homeserver, type HomeserverOptions, type Logger, M, MatrixError, type MediaAdapter, type MediaMetadata, type ModelDef, type MysqlPoolLike, type NodeHandler, type OAuthDelegationOptions, type Operator, type PasswordCheckContext, type PgClusterPoolLike, type PgPoolLike, type RateLimitAction, type RateLimitRule, type RateLimiter, type Row, type RtcOptions, type S3MediaOptions, type Schema, type ServerNoticesOptions, type SqlAdapterOptions, type SqlDialect, type SqlExecutor, type SqliteDatabaseLike, type SsoOptions, type SsoProvider, type ThreepidEmail, type ThreepidOptions, type ThreepidPurpose, type ThreepidSms, type ThumbnailRequest, type Thumbnailer, type TurnOptions, UNSTABLE, type UserAuthAdapter, type WhereClause, createHomeserver, filesystemMedia, getExpressHandler, getFastifyPlugin, getFetchHandler, getHonoHandler, getNodeHandler, handleRequest, isMatrixPath, loadAppServiceRegistrations, localCluster, memoryAdapter, memoryMedia, memoryRateLimiter, mysqlAdapter, parseAppServiceRegistration, postgresAdapter, postgresCluster, s3Media, schema, sharpThumbnailer, sqlAdapter, sqliteAdapter };

package/dist/index.js

@@ -11,5 +11,5 @@ import { Homeserver, createHomeserver } from "./homeserver.js";
 import { getExpressHandler, getFastifyPlugin, getFetchHandler, getHonoHandler, getNodeHandler, isMatrixPath } from "./http/adapters.js";
 import { mysqlAdapter, postgresAdapter, sqlAdapter, sqliteAdapter } from "./storage/sql.js";
 import { s3Media } from "./adapters/s3.js";
-import { postgresCluster, redisCluster } from "./adapters/cluster.js";
-export { Homeserver, M, MatrixError, UNSTABLE, createHomeserver, filesystemMedia, getExpressHandler, getFastifyPlugin, getFetchHandler, getHonoHandler, getNodeHandler, handleRequest, isMatrixPath, loadAppServiceRegistrations, localCluster, memoryAdapter, memoryMedia, memoryRateLimiter, mysqlAdapter, parseAppServiceRegistration, postgresAdapter, postgresCluster, redisCluster, s3Media, schema, sharpThumbnailer, sqlAdapter, sqliteAdapter };
+import { postgresCluster } from "./adapters/cluster.js";
+export { Homeserver, M, MatrixError, UNSTABLE, createHomeserver, filesystemMedia, getExpressHandler, getFastifyPlugin, getFetchHandler, getHonoHandler, getNodeHandler, handleRequest, isMatrixPath, loadAppServiceRegistrations, localCluster, memoryAdapter, memoryMedia, memoryRateLimiter, mysqlAdapter, parseAppServiceRegistration, postgresAdapter, postgresCluster, s3Media, schema, sharpThumbnailer, sqlAdapter, sqliteAdapter };

package/dist/storage/batch.js

@@ -0,0 +1,82 @@
+//#region src/storage/batch.ts
+/**
+* Run independent reads together: in one round trip when the adapter can (`findManyBatch`),
+* otherwise concurrently. Results come in the order of the queries.
+*/
+function readBatch(db, queries) {
+	if (db.findManyBatch) return db.findManyBatch(queries);
+	return Promise.all(queries.map((query) => db.findMany(query.model, query.opts)));
+}
+/** Upsert several rows: in a few statements when the adapter can (`upsertMany`), otherwise one by one. */
+async function upsertMany(db, model, rows) {
+	if (rows.length === 0) return;
+	if (db.upsertMany) return db.upsertMany(model, rows);
+	for (const row of rows) await db.upsert(model, row);
+}
+/** At most this many reads go into one coalesced statement. */
+const MAX_COALESCED = 32;
+/**
+* An adapter whose reads (`findMany`, `findOne`) issued within one turn of the event loop are
+* sent together, through `findManyBatch`. Writes and transactions go straight to `db`. Adapters
+* without `findManyBatch` are returned as they are.
+*
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/43: under load, most of the homeserver's
+* and the database's CPU went to the per-statement cost (a socket write, a round trip, parsing)
+* of many small reads issued at the same moment by concurrent requests. Nothing is kept between
+* turns: this merges requests, it does not cache results. A read waits for at most the rest of
+* the current turn. If a coalesced statement fails, its reads are run one by one, so a bad read
+* fails alone.
+*/
+function coalesceReads(db) {
+	if (!db.findManyBatch) return db;
+	let queue = [];
+	const runAlone = (read) => db.findMany(read.query.model, read.query.opts).then(read.resolve, read.reject);
+	const run = async (reads) => {
+		if (reads.length === 1) return runAlone(reads[0]);
+		let results;
+		try {
+			results = await db.findManyBatch(reads.map((r) => r.query), { prepare: false });
+		} catch {
+			for (const read of reads) runAlone(read);
+			return;
+		}
+		reads.forEach((read, i) => read.resolve(results[i]));
+	};
+	const flush = () => {
+		const reads = queue;
+		queue = [];
+		for (let i = 0; i < reads.length; i += MAX_COALESCED) run(reads.slice(i, i + MAX_COALESCED));
+	};
+	const enqueue = (query) => new Promise((resolve, reject) => {
+		if (queue.length === 0) setImmediate(flush);
+		queue.push({
+			query,
+			resolve,
+			reject
+		});
+	});
+	return {
+		findMany: (model, opts) => enqueue({
+			model,
+			opts
+		}),
+		findOne: async (model, where) => (await enqueue({
+			model,
+			opts: {
+				where,
+				limit: 1
+			}
+		}))[0] ?? null,
+		findManyBatch: (queries, options) => db.findManyBatch(queries, options),
+		create: (model, data) => db.create(model, data),
+		upsert: (model, data) => db.upsert(model, data),
+		...db.upsertMany ? { upsertMany: (model, rows) => db.upsertMany(model, rows) } : {},
+		update: (model, where, data) => db.update(model, where, data),
+		delete: (model, where) => db.delete(model, where),
+		max: (model, field) => db.max(model, field),
+		...db.migrate ? { migrate: (schema) => db.migrate(schema) } : {},
+		...db.transaction ? { transaction: (fn) => db.transaction(fn) } : {}
+	};
+}
+//#endregion
+export { coalesceReads, readBatch, upsertMany };

package/dist/storage/memory.js

@@ -72,6 +72,7 @@ function memoryAdapter() {
 					rows.sort((a, b) => a[field] < b[field] ? -dir : a[field] > b[field] ? dir : 0);
 				}
 				if (opts.limit !== void 0) rows = rows.slice(0, opts.limit);
+				if (opts.fields?.length) return rows.map((r) => Object.fromEntries(opts.fields.map((f) => [f, structuredClone(r[f] ?? null)])));
 				return rows.map((r) => structuredClone(r));
 			},
 			async update(model, where, data) {