matrix-nodejs-homeserver 0.3.1 → 0.3.3

package/dist/storage/schema.js

@@ -192,7 +192,11 @@ const schema = {
 			ts: n
 		},
 		primaryKey: ["user_id", "event_id"],
-		indexes: [["user_id", "room_id"], ["user_id", "stream"]]
+		indexes: [[
+			"user_id",
+			"room_id",
+			"stream"
+		], ["user_id", "stream"]]
 	},
 	sso_identities: {
 		fields: {
@@ -249,7 +253,7 @@ const schema = {
 			forgotten: b
 		},
 		primaryKey: ["room_id", "user_id"],
-		indexes: [["user_id"]]
+		indexes: [["user_id"], ["stream"]]
 	},
 	room_aliases: {
 		fields: {
@@ -397,7 +401,8 @@ const schema = {
 			last_active_ts: n,
 			stream: n
 		},
-		primaryKey: ["user_id"]
+		primaryKey: ["user_id"],
+		indexes: [["stream"]]
 	},
 	media: {
 		fields: {

package/dist/storage/sql.d.ts

@@ -2,8 +2,10 @@ import { DatabaseAdapter, Row } from "./types.js";
 //#region src/storage/sql.d.ts
 export type SqlDialect = "sqlite" | "postgres" | "mysql";
 export interface SqlExecutor {
-  /** Run a query that returns rows. */
-  query(sql: string, params: unknown[]): Promise<Row[]>;
+  /** Run a query that returns rows; `prepare: false` when the statement is not worth preparing for reuse. */
+  query(sql: string, params: unknown[], options?: {
+    prepare?: boolean;
+  }): Promise<Row[]>;
   /** Run a statement; resolve with the number of affected rows. */
   execute(sql: string, params: unknown[]): Promise<number>;
 }
@@ -36,9 +38,20 @@ export interface SqliteDatabaseLike {
 /** SQLite adapter for `node:sqlite` (`new DatabaseSync(path)`) or `better-sqlite3`. */
 export declare function sqliteAdapter(db: SqliteDatabaseLike, options?: {
   tablePrefix?: string;
+  /**
+   * When commits reach the disk (SQLite's `synchronous`). `"normal"` (the default): at
+   * checkpoints; a commit survives a crash of the process, and a power loss or operating
+   * system crash can roll back the most recent ones, without corrupting the database.
+   * `"full"`: on every commit, which blocks the event loop for an fsync each time.
+   */
+  synchronous?: "normal" | "full";
 }): DatabaseAdapter;
 interface PgQueryable {
-  query(text: string, values?: unknown[]): Promise<{
+  query(config: {
+    text: string;
+    values?: unknown[];
+    name?: string;
+  }): Promise<{
     rows: Row[];
     rowCount: number | null;
   }>;
@@ -53,6 +66,12 @@ export interface PgPoolLike extends PgQueryable {
 /** PostgreSQL adapter for `pg` (node-postgres). Pass a `Pool` to get transactions. */
 export declare function postgresAdapter(pool: PgPoolLike, options?: {
   tablePrefix?: string;
+  /**
+   * Prepare statements on each connection and reuse them (default true). Turn off behind a
+   * connection pooler that does not support prepared statements, such as PgBouncer before 1.21
+   * or in transaction mode without `max_prepared_statements`.
+   */
+  preparedStatements?: boolean;
 }): DatabaseAdapter;
 interface MysqlQueryable {
   query(sql: string, values?: unknown[]): Promise<[any, unknown]>;

package/dist/storage/sql.js

@@ -1,5 +1,8 @@
+import { createHash } from "node:crypto";
 import { AsyncLocalStorage } from "node:async_hooks";
 //#region src/storage/sql.ts
+/** Parameters per statement for multi-row writes: within SQLite's (32766), Postgres' and MySQL's (65535) limits. */
+const MAX_PARAMS = 4e3;
 const OPS = {
 	eq: "=",
 	ne: "<>",
@@ -60,6 +63,12 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 		return out;
 	};
 	const placeholder = (i) => dialect === "postgres" ? `$${i}` : "?";
+	const limitClause = (limit, params) => {
+		const n = Math.max(0, Math.floor(limit));
+		if (dialect === "mysql") return ` LIMIT ${n}`;
+		params.push(n);
+		return ` LIMIT ${placeholder(params.length)}`;
+	};
 	const buildWhere = (model, where = [], params) => {
 		if (where.length === 0) return "";
 		return ` WHERE ${where.map((w) => {
@@ -67,6 +76,10 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 			if (op === "in") {
 				const values = w.value;
 				if (values.length === 0) return "1=0";
+				if (dialect === "postgres") {
+					params.push(values.map((v) => toDb(model, w.field, v)));
+					return `${q(w.field)} = ANY(${placeholder(params.length)})`;
+				}
 				const ph = values.map((v) => {
 					params.push(toDb(model, w.field, v));
 					return placeholder(params.length);
@@ -83,11 +96,15 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 	};
 	const build = (executor, inTx) => {
 		const insert = async (model, data, mode) => {
-			const fields = Object.keys(data).filter((k) => data[k] !== void 0);
-			const params = fields.map((f) => toDb(model, f, data[f]));
+			const rows = Array.isArray(data) ? data : [data];
+			const fields = Object.keys(rows[0]).filter((k) => rows[0][k] !== void 0);
+			const params = [];
+			const values = rows.map((row) => `(${fields.map((f) => {
+				params.push(toDb(model, f, row[f]));
+				return placeholder(params.length);
+			}).join(", ")})`);
 			const cols = fields.map(q).join(", ");
-			const ph = fields.map((_, i) => placeholder(i + 1)).join(", ");
-			let sql = `INSERT INTO ${t(model)} (${cols}) VALUES (${ph})`;
+			let sql = `INSERT INTO ${t(model)} (${cols}) VALUES ${values.join(", ")}`;
 			if (mode === "upsert") {
 				const pk = schema[model]?.primaryKey ?? [];
 				const rest = fields.filter((f) => !pk.includes(f));
@@ -123,6 +140,20 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 			},
 			create: (model, data) => insert(model, data, "insert"),
 			upsert: (model, data) => insert(model, data, "upsert"),
+			async upsertMany(model, rows) {
+				const pk = schema[model]?.primaryKey ?? [];
+				const byKey = /* @__PURE__ */ new Map();
+				for (const row of rows) byKey.set(JSON.stringify(pk.map((f) => row[f])), row);
+				const groups = /* @__PURE__ */ new Map();
+				for (const row of byKey.values()) {
+					const shape = Object.keys(row).filter((k) => row[k] !== void 0).join("\0");
+					groups.set(shape, [...groups.get(shape) ?? [], row]);
+				}
+				for (const group of groups.values()) {
+					const perStatement = Math.max(1, Math.floor(MAX_PARAMS / Object.keys(group[0]).length));
+					for (let i = 0; i < group.length; i += perStatement) await insert(model, group.slice(i, i + perStatement), "upsert");
+				}
+			},
 			async findOne(model, where) {
 				const params = [];
 				const rows = await executor.query(`SELECT * FROM ${t(model)}${buildWhere(model, where, params)} LIMIT 1`, params);
@@ -130,11 +161,31 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 			},
 			async findMany(model, opts = {}) {
 				const params = [];
-				let sql = `SELECT * FROM ${t(model)}${buildWhere(model, opts.where, params)}`;
+				let sql = `SELECT ${opts.fields?.length ? opts.fields.map(q).join(", ") : "*"} FROM ${t(model)}${buildWhere(model, opts.where, params)}`;
 				if (opts.sortBy) sql += ` ORDER BY ${q(opts.sortBy.field)} ${opts.sortBy.direction === "asc" ? "ASC" : "DESC"}`;
-				if (opts.limit !== void 0) sql += ` LIMIT ${Math.max(0, Math.floor(opts.limit))}`;
+				if (opts.limit !== void 0) sql += limitClause(opts.limit, params);
 				return (await executor.query(sql, params)).map((r) => fromDb(model, r));
 			},
+			...dialect === "postgres" ? { async findManyBatch(queries, options) {
+				if (queries.length === 0) return [];
+				const params = [];
+				const parts = queries.map(({ model, opts = {} }, i) => {
+					const fields = opts.fields?.length && opts.sortBy && !opts.fields.includes(opts.sortBy.field) ? [...opts.fields, opts.sortBy.field] : opts.fields;
+					let inner = `SELECT ${fields?.length ? fields.map(q).join(", ") : "*"} FROM ${t(model)}${buildWhere(model, opts.where, params)}`;
+					const direction = opts.sortBy?.direction === "asc" ? "ASC" : "DESC";
+					if (opts.sortBy) inner += ` ORDER BY ${q(opts.sortBy.field)} ${direction}`;
+					if (opts.limit !== void 0) inner += limitClause(opts.limit, params);
+					return `SELECT ${i} AS i, (SELECT json_agg(b${opts.sortBy ? ` ORDER BY b.${q(opts.sortBy.field)} ${direction}` : ""}) FROM (${inner}) b) AS r`;
+				});
+				const rows = await executor.query(parts.join(" UNION ALL "), params, options);
+				const out = queries.map(() => []);
+				for (const row of rows) {
+					const list = typeof row.r === "string" ? JSON.parse(row.r) : row.r;
+					const model = queries[Number(row.i)].model;
+					out[Number(row.i)] = (list ?? []).map((r) => fromDb(model, r));
+				}
+				return out;
+			} } : {},
 			async update(model, where, data) {
 				const fields = Object.keys(data).filter((k) => data[k] !== void 0);
 				if (fields.length === 0) return 0;
@@ -156,21 +207,36 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 	};
 	return build(rootExecutor, false);
 }
+/** How long SQLite statements may run back to back before yielding to the event loop (#47). */
+const YIELD_AFTER_MS = 10;
+/** Prepared statements an SQLite adapter keeps (#40). */
+const STATEMENT_CACHE = 500;
 /** SQLite adapter for `node:sqlite` (`new DatabaseSync(path)`) or `better-sqlite3`. */
 function sqliteAdapter(db, options = {}) {
 	try {
 		db.exec("PRAGMA journal_mode = WAL");
+		db.exec(`PRAGMA synchronous = ${options.synchronous === "full" ? "FULL" : "NORMAL"}`);
 	} catch {}
 	const cache = /* @__PURE__ */ new Map();
 	const stmt = (sql) => {
 		let s = cache.get(sql);
-		if (!s) cache.set(sql, s = db.prepare(sql));
+		if (s) cache.delete(sql);
+		else s = db.prepare(sql);
+		cache.set(sql, s);
+		if (cache.size > STATEMENT_CACHE) cache.delete(cache.keys().next().value);
 		return s;
 	};
 	const inTransaction = new AsyncLocalStorage();
 	let gate = Promise.resolve();
+	let lastYield = performance.now();
 	const waitGate = async () => {
-		if (!inTransaction.getStore()) await gate;
+		if (inTransaction.getStore()) return;
+		await gate;
+		if (performance.now() - lastYield > YIELD_AFTER_MS) {
+			await new Promise((resolve) => setImmediate(resolve));
+			lastYield = performance.now();
+			await gate;
+		}
 	};
 	const executor = {
 		async query(sql, params) {
@@ -210,29 +276,80 @@ function sqliteAdapter(db, options = {}) {
 		}
 	});
 }
-const pgExecutor = (conn) => ({
-	async query(sql, params) {
-		return (await conn.query(sql, params)).rows;
-	},
-	async execute(sql, params) {
-		return (await conn.query(sql, params)).rowCount ?? 0;
-	}
-});
+/**
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/40: unnamed statements are parsed and
+* planned by Postgres on every call, which took most of its CPU under load. Named ones are prepared
+* once per connection and reused. Statement texts depend only on the code (lists and limits are
+* parameters), so there are a few hundred of them; past `MAX_PREPARED` distinct texts, further
+* ones run unnamed, which bounds the memory both node-postgres and Postgres keep per connection.
+*/
+const MAX_PREPARED = 2e3;
+function statementNamer(enabled) {
+	let names = /* @__PURE__ */ new Map();
+	let generation = 0;
+	return {
+		name(sql) {
+			if (!enabled) return void 0;
+			let name = names.get(sql);
+			if (!name && names.size < MAX_PREPARED) names.set(sql, name = `m${generation}_${createHash("sha1").update(sql).digest("base64url")}`);
+			return name;
+		},
+		reset() {
+			names = /* @__PURE__ */ new Map();
+			generation++;
+		}
+	};
+}
+/**
+* A prepared `SELECT *` fails once its table has gained a column, for example when another node of
+* a cluster migrated the schema during a rolling update. Then the statements are prepared again
+* under new names and the query is retried once.
+*/
+const isStalePlan = (err) => err?.code === "0A000" && /cached plan/.test(String(err.message));
+const pgExecutor = (conn, namer) => {
+	const run = async (sql, params, prepare = true) => {
+		const name = () => prepare ? namer.name(sql) : void 0;
+		try {
+			return await conn.query({
+				text: sql,
+				values: params,
+				name: name()
+			});
+		} catch (err) {
+			if (!isStalePlan(err)) throw err;
+			namer.reset();
+			return conn.query({
+				text: sql,
+				values: params,
+				name: name()
+			});
+		}
+	};
+	return {
+		async query(sql, params, options) {
+			return (await run(sql, params, options?.prepare)).rows;
+		},
+		async execute(sql, params) {
+			return (await run(sql, params)).rowCount ?? 0;
+		}
+	};
+};
 /** PostgreSQL adapter for `pg` (node-postgres). Pass a `Pool` to get transactions. */
 function postgresAdapter(pool, options = {}) {
+	const namer = statementNamer(options.preparedStatements ?? true);
 	return sqlAdapter({
 		dialect: "postgres",
 		tablePrefix: options.tablePrefix,
-		executor: pgExecutor(pool),
+		executor: pgExecutor(pool, namer),
 		transaction: pool.connect ? async (fn) => {
 			const client = await pool.connect();
 			try {
-				await client.query("BEGIN");
-				const result = await fn(pgExecutor(client));
-				await client.query("COMMIT");
+				await client.query({ text: "BEGIN" });
+				const result = await fn(pgExecutor(client, namer));
+				await client.query({ text: "COMMIT" });
 				return result;
 			} catch (err) {
-				await client.query("ROLLBACK").catch(() => {});
+				await client.query({ text: "ROLLBACK" }).catch(() => {});
 				throw err;
 			} finally {
 				client.release();

package/dist/storage/types.d.ts

@@ -21,8 +21,26 @@ export interface WhereClause {
   operator?: Operator;
 }
 export type Row = Record<string, any>;
+/** How a `findManyBatch` is run. */
+export interface BatchOptions {
+  /**
+   * The combination of reads is a one-off (as when unrelated reads are coalesced), so preparing
+   * the statement for reuse would only fill the database's statement cache. Default true.
+   */
+  prepare?: boolean;
+}
+/** One read of a `findManyBatch`. */
+export interface BatchQuery {
+  model: string;
+  opts?: FindManyOptions;
+}
 export interface FindManyOptions {
   where?: WhereClause[];
+  /**
+   * Only these fields are needed. Adapters should read just these (a smaller query and less to
+   * parse); one that ignores the option and returns whole rows is still correct.
+   */
+  fields?: string[];
   sortBy?: {
     field: string;
     direction: "asc" | "desc";
@@ -35,11 +53,23 @@ export interface FindManyOptions {
  * All `where` clauses are combined with AND.
  */
 export interface DatabaseAdapter {
+  /**
+   * Several independent reads in one go: the results in the same order as the queries. Optional;
+   * an adapter that talks to a database server can send them in one round trip. Without it,
+   * callers run the reads separately (see `readBatch`).
+   */
+  findManyBatch?(queries: BatchQuery[], options?: BatchOptions): Promise<Row[][]>;
   /** Create tables / collections for the given schema. Must be idempotent. */
   migrate?(schema: Schema): Promise<void>;
   create(model: string, data: Row): Promise<void>;
   /** Insert or replace by primary key. */
   upsert(model: string, data: Row): Promise<void>;
+  /**
+   * `upsert` for several rows at once (the last row wins when two share a primary key). Optional;
+   * an adapter that talks to a database server can write them in a few statements. Without it,
+   * callers upsert the rows one by one (see `upsertMany`).
+   */
+  upsertMany?(model: string, rows: Row[]): Promise<void>;
   findOne(model: string, where: WhereClause[]): Promise<Row | null>;
   findMany(model: string, options?: FindManyOptions): Promise<Row[]>;
   update(model: string, where: WhereClause[], data: Row): Promise<number>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-nodejs-homeserver",
-  "version": "0.3.1",
+  "version": "0.3.3",
   "description": "A framework- and database-agnostic Matrix homeserver library for Node.js",
   "keywords": [
     "chat",
@@ -41,10 +41,12 @@
   },
   "scripts": {
     "typecheck": "tsc -p tsconfig.json",
+    "test": "node --test \"test/**/*.test.ts\"",
     "lint": "oxlint",
     "format": "oxfmt .",
     "format:check": "oxfmt --check .",
     "build": "tsdown",
+    "check:package": "publint && attw --pack . --profile esm-only",
     "prepare": "npm run build",
     "start:example": "node examples/node-http.ts",
     "start:multi-node-example": "node examples/multi-node.ts",
@@ -53,15 +55,17 @@
     "release": "changeset publish"
   },
   "devDependencies": {
+    "@arethetypeswrong/cli": "^0.18.5",
     "@changesets/cli": "^3.0.3",
     "@types/node": "^26.6.5",
     "oxfmt": "^0.73.0",
     "oxlint": "^1.88.0",
+    "publint": "^0.3.25",
     "tsdown": "^0.23.1",
     "typescript": "7.0.2",
     "zod": "^4.6.7"
   },
   "engines": {
-    "node": ">=26"
+    "node": ">=22"
   }
 }