matrix-nodejs-homeserver 0.3.2 → 0.3.4

package/dist/storage/sql.js

@@ -1,5 +1,8 @@
+import { createHash } from "node:crypto";
 import { AsyncLocalStorage } from "node:async_hooks";
 //#region src/storage/sql.ts
+/** Parameters per statement for multi-row writes: within SQLite's (32766), Postgres' and MySQL's (65535) limits. */
+const MAX_PARAMS = 4e3;
 const OPS = {
 	eq: "=",
 	ne: "<>",
@@ -60,6 +63,12 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 		return out;
 	};
 	const placeholder = (i) => dialect === "postgres" ? `$${i}` : "?";
+	const limitClause = (limit, params) => {
+		const n = Math.max(0, Math.floor(limit));
+		if (dialect === "mysql") return ` LIMIT ${n}`;
+		params.push(n);
+		return ` LIMIT ${placeholder(params.length)}`;
+	};
 	const buildWhere = (model, where = [], params) => {
 		if (where.length === 0) return "";
 		return ` WHERE ${where.map((w) => {
@@ -67,6 +76,10 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 			if (op === "in") {
 				const values = w.value;
 				if (values.length === 0) return "1=0";
+				if (dialect === "postgres") {
+					params.push(values.map((v) => toDb(model, w.field, v)));
+					return `${q(w.field)} = ANY(${placeholder(params.length)})`;
+				}
 				const ph = values.map((v) => {
 					params.push(toDb(model, w.field, v));
 					return placeholder(params.length);
@@ -83,11 +96,15 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 	};
 	const build = (executor, inTx) => {
 		const insert = async (model, data, mode) => {
-			const fields = Object.keys(data).filter((k) => data[k] !== void 0);
-			const params = fields.map((f) => toDb(model, f, data[f]));
+			const rows = Array.isArray(data) ? data : [data];
+			const fields = Object.keys(rows[0]).filter((k) => rows[0][k] !== void 0);
+			const params = [];
+			const values = rows.map((row) => `(${fields.map((f) => {
+				params.push(toDb(model, f, row[f]));
+				return placeholder(params.length);
+			}).join(", ")})`);
 			const cols = fields.map(q).join(", ");
-			const ph = fields.map((_, i) => placeholder(i + 1)).join(", ");
-			let sql = `INSERT INTO ${t(model)} (${cols}) VALUES (${ph})`;
+			let sql = `INSERT INTO ${t(model)} (${cols}) VALUES ${values.join(", ")}`;
 			if (mode === "upsert") {
 				const pk = schema[model]?.primaryKey ?? [];
 				const rest = fields.filter((f) => !pk.includes(f));
@@ -123,6 +140,20 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 			},
 			create: (model, data) => insert(model, data, "insert"),
 			upsert: (model, data) => insert(model, data, "upsert"),
+			async upsertMany(model, rows) {
+				const pk = schema[model]?.primaryKey ?? [];
+				const byKey = /* @__PURE__ */ new Map();
+				for (const row of rows) byKey.set(JSON.stringify(pk.map((f) => row[f])), row);
+				const groups = /* @__PURE__ */ new Map();
+				for (const row of byKey.values()) {
+					const shape = Object.keys(row).filter((k) => row[k] !== void 0).join("\0");
+					groups.set(shape, [...groups.get(shape) ?? [], row]);
+				}
+				for (const group of groups.values()) {
+					const perStatement = Math.max(1, Math.floor(MAX_PARAMS / Object.keys(group[0]).length));
+					for (let i = 0; i < group.length; i += perStatement) await insert(model, group.slice(i, i + perStatement), "upsert");
+				}
+			},
 			async findOne(model, where) {
 				const params = [];
 				const rows = await executor.query(`SELECT * FROM ${t(model)}${buildWhere(model, where, params)} LIMIT 1`, params);
@@ -132,25 +163,26 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 				const params = [];
 				let sql = `SELECT ${opts.fields?.length ? opts.fields.map(q).join(", ") : "*"} FROM ${t(model)}${buildWhere(model, opts.where, params)}`;
 				if (opts.sortBy) sql += ` ORDER BY ${q(opts.sortBy.field)} ${opts.sortBy.direction === "asc" ? "ASC" : "DESC"}`;
-				if (opts.limit !== void 0) sql += ` LIMIT ${Math.max(0, Math.floor(opts.limit))}`;
+				if (opts.limit !== void 0) sql += limitClause(opts.limit, params);
 				return (await executor.query(sql, params)).map((r) => fromDb(model, r));
 			},
-			...dialect === "postgres" ? { async findManyBatch(queries) {
+			...dialect === "postgres" ? { async findManyBatch(queries, options) {
 				if (queries.length === 0) return [];
 				const params = [];
 				const parts = queries.map(({ model, opts = {} }, i) => {
 					const fields = opts.fields?.length && opts.sortBy && !opts.fields.includes(opts.sortBy.field) ? [...opts.fields, opts.sortBy.field] : opts.fields;
 					let inner = `SELECT ${fields?.length ? fields.map(q).join(", ") : "*"} FROM ${t(model)}${buildWhere(model, opts.where, params)}`;
-					const order = opts.sortBy ? `${q(opts.sortBy.field)} ${opts.sortBy.direction === "asc" ? "ASC" : "DESC"}` : "";
-					if (order) inner += ` ORDER BY ${order}`;
-					if (opts.limit !== void 0) inner += ` LIMIT ${Math.max(0, Math.floor(opts.limit))}`;
-					return `SELECT ${i} AS i, row_number() OVER (${order ? `ORDER BY ${order.replace(/^"/, "b.\"")}` : ""}) AS n, to_jsonb(b) AS r FROM (${inner}) b`;
+					const direction = opts.sortBy?.direction === "asc" ? "ASC" : "DESC";
+					if (opts.sortBy) inner += ` ORDER BY ${q(opts.sortBy.field)} ${direction}`;
+					if (opts.limit !== void 0) inner += limitClause(opts.limit, params);
+					return `SELECT ${i} AS i, (SELECT json_agg(b${opts.sortBy ? ` ORDER BY b.${q(opts.sortBy.field)} ${direction}` : ""}) FROM (${inner}) b) AS r`;
 				});
-				const rows = await executor.query(`${parts.join(" UNION ALL ")} ORDER BY i, n`, params);
+				const rows = await executor.query(parts.join(" UNION ALL "), params, options);
 				const out = queries.map(() => []);
 				for (const row of rows) {
-					const r = typeof row.r === "string" ? JSON.parse(row.r) : row.r;
-					out[Number(row.i)].push(fromDb(queries[Number(row.i)].model, r));
+					const list = typeof row.r === "string" ? JSON.parse(row.r) : row.r;
+					const model = queries[Number(row.i)].model;
+					out[Number(row.i)] = (list ?? []).map((r) => fromDb(model, r));
 				}
 				return out;
 			} } : {},
@@ -175,21 +207,36 @@ function sqlAdapter({ dialect, executor: rootExecutor, tablePrefix = "", transac
 	};
 	return build(rootExecutor, false);
 }
+/** How long SQLite statements may run back to back before yielding to the event loop (#47, #52). */
+const YIELD_AFTER_MS = 50;
+/** Prepared statements an SQLite adapter keeps (#40). */
+const STATEMENT_CACHE = 500;
 /** SQLite adapter for `node:sqlite` (`new DatabaseSync(path)`) or `better-sqlite3`. */
 function sqliteAdapter(db, options = {}) {
 	try {
 		db.exec("PRAGMA journal_mode = WAL");
+		db.exec(`PRAGMA synchronous = ${options.synchronous === "full" ? "FULL" : "NORMAL"}`);
 	} catch {}
 	const cache = /* @__PURE__ */ new Map();
 	const stmt = (sql) => {
 		let s = cache.get(sql);
-		if (!s) cache.set(sql, s = db.prepare(sql));
+		if (s) cache.delete(sql);
+		else s = db.prepare(sql);
+		cache.set(sql, s);
+		if (cache.size > STATEMENT_CACHE) cache.delete(cache.keys().next().value);
 		return s;
 	};
 	const inTransaction = new AsyncLocalStorage();
 	let gate = Promise.resolve();
+	let lastYield = performance.now();
 	const waitGate = async () => {
-		if (!inTransaction.getStore()) await gate;
+		if (inTransaction.getStore()) return;
+		await gate;
+		if (performance.now() - lastYield > YIELD_AFTER_MS) {
+			await new Promise((resolve) => setImmediate(resolve));
+			lastYield = performance.now();
+			await gate;
+		}
 	};
 	const executor = {
 		async query(sql, params) {
@@ -229,29 +276,80 @@ function sqliteAdapter(db, options = {}) {
 		}
 	});
 }
-const pgExecutor = (conn) => ({
-	async query(sql, params) {
-		return (await conn.query(sql, params)).rows;
-	},
-	async execute(sql, params) {
-		return (await conn.query(sql, params)).rowCount ?? 0;
-	}
-});
+/**
+* https://github.com/k18k/matrix-nodejs-homeserver/issues/40: unnamed statements are parsed and
+* planned by Postgres on every call, which took most of its CPU under load. Named ones are prepared
+* once per connection and reused. Statement texts depend only on the code (lists and limits are
+* parameters), so there are a few hundred of them; past `MAX_PREPARED` distinct texts, further
+* ones run unnamed, which bounds the memory both node-postgres and Postgres keep per connection.
+*/
+const MAX_PREPARED = 2e3;
+function statementNamer(enabled) {
+	let names = /* @__PURE__ */ new Map();
+	let generation = 0;
+	return {
+		name(sql) {
+			if (!enabled) return void 0;
+			let name = names.get(sql);
+			if (!name && names.size < MAX_PREPARED) names.set(sql, name = `m${generation}_${createHash("sha1").update(sql).digest("base64url")}`);
+			return name;
+		},
+		reset() {
+			names = /* @__PURE__ */ new Map();
+			generation++;
+		}
+	};
+}
+/**
+* A prepared `SELECT *` fails once its table has gained a column, for example when another node of
+* a cluster migrated the schema during a rolling update. Then the statements are prepared again
+* under new names and the query is retried once.
+*/
+const isStalePlan = (err) => err?.code === "0A000" && /cached plan/.test(String(err.message));
+const pgExecutor = (conn, namer) => {
+	const run = async (sql, params, prepare = true) => {
+		const name = () => prepare ? namer.name(sql) : void 0;
+		try {
+			return await conn.query({
+				text: sql,
+				values: params,
+				name: name()
+			});
+		} catch (err) {
+			if (!isStalePlan(err)) throw err;
+			namer.reset();
+			return conn.query({
+				text: sql,
+				values: params,
+				name: name()
+			});
+		}
+	};
+	return {
+		async query(sql, params, options) {
+			return (await run(sql, params, options?.prepare)).rows;
+		},
+		async execute(sql, params) {
+			return (await run(sql, params)).rowCount ?? 0;
+		}
+	};
+};
 /** PostgreSQL adapter for `pg` (node-postgres). Pass a `Pool` to get transactions. */
 function postgresAdapter(pool, options = {}) {
+	const namer = statementNamer(options.preparedStatements ?? true);
 	return sqlAdapter({
 		dialect: "postgres",
 		tablePrefix: options.tablePrefix,
-		executor: pgExecutor(pool),
+		executor: pgExecutor(pool, namer),
 		transaction: pool.connect ? async (fn) => {
 			const client = await pool.connect();
 			try {
-				await client.query("BEGIN");
-				const result = await fn(pgExecutor(client));
-				await client.query("COMMIT");
+				await client.query({ text: "BEGIN" });
+				const result = await fn(pgExecutor(client, namer));
+				await client.query({ text: "COMMIT" });
 				return result;
 			} catch (err) {
-				await client.query("ROLLBACK").catch(() => {});
+				await client.query({ text: "ROLLBACK" }).catch(() => {});
 				throw err;
 			} finally {
 				client.release();

package/dist/storage/types.d.ts

@@ -21,6 +21,14 @@ export interface WhereClause {
   operator?: Operator;
 }
 export type Row = Record<string, any>;
+/** How a `findManyBatch` is run. */
+export interface BatchOptions {
+  /**
+   * The combination of reads is a one-off (as when unrelated reads are coalesced), so preparing
+   * the statement for reuse would only fill the database's statement cache. Default true.
+   */
+  prepare?: boolean;
+}
 /** One read of a `findManyBatch`. */
 export interface BatchQuery {
   model: string;
@@ -50,12 +58,18 @@ export interface DatabaseAdapter {
    * an adapter that talks to a database server can send them in one round trip. Without it,
    * callers run the reads separately (see `readBatch`).
    */
-  findManyBatch?(queries: BatchQuery[]): Promise<Row[][]>;
+  findManyBatch?(queries: BatchQuery[], options?: BatchOptions): Promise<Row[][]>;
   /** Create tables / collections for the given schema. Must be idempotent. */
   migrate?(schema: Schema): Promise<void>;
   create(model: string, data: Row): Promise<void>;
   /** Insert or replace by primary key. */
   upsert(model: string, data: Row): Promise<void>;
+  /**
+   * `upsert` for several rows at once (the last row wins when two share a primary key). Optional;
+   * an adapter that talks to a database server can write them in a few statements. Without it,
+   * callers upsert the rows one by one (see `upsertMany`).
+   */
+  upsertMany?(model: string, rows: Row[]): Promise<void>;
   findOne(model: string, where: WhereClause[]): Promise<Row | null>;
   findMany(model: string, options?: FindManyOptions): Promise<Row[]>;
   update(model: string, where: WhereClause[], data: Row): Promise<number>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-nodejs-homeserver",
-  "version": "0.3.2",
+  "version": "0.3.4",
   "description": "A framework- and database-agnostic Matrix homeserver library for Node.js",
   "keywords": [
     "chat",