@jskit-ai/database-runtime 0.1.260 → 0.1.261

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@jskit-ai/database-runtime",
3
- "version": "0.1.260",
3
+ "version": "0.1.261",
4
4
  "type": "module",
5
5
  "scripts": {
6
6
  "test": "jskit test"
@@ -10,6 +10,7 @@
10
10
  "./server/databaseSetup": "./src/server/databaseSetup.js",
11
11
  "./server/knexMigrationConfig": "./src/server/knexMigrationConfig.js",
12
12
  "./server/foreignKeyMigrations": "./src/server/foreignKeyMigrations.js",
13
+ "./server/testDatabases": "./src/server/testDatabases.js",
13
14
  "./server/types": "./src/server/types.js",
14
15
  "./shared": "./src/shared/index.js",
15
16
  "./shared/dateUtils": "./src/shared/dateUtils.js",
@@ -65,6 +66,10 @@
65
66
  "subpath": "./server/foreignKeyMigrations",
66
67
  "summary": "Adds a single-column foreign key only when it is missing, refusing rows without a parent, for constraints migrations on fresh and already-migrated databases."
67
68
  },
69
+ {
70
+ "subpath": "./server/testDatabases",
71
+ "summary": "Gives each test file or browser worker running at once its own disposable copy of TEST_DB_NAME or BROWSER_TEST_DB_NAME, restored from a schema snapshot keyed by the migrations' names and contents (MySQL, MariaDB and PostgreSQL), and counts the statements a piece of work sends, for query-count guards."
72
+ },
68
73
  {
69
74
  "subpath": "./shared",
70
75
  "summary": "Exports shared repository helpers (transactions, record ids, JSON columns, visibility, duplicate-entry detection and date/time serialization)."
@@ -78,6 +83,6 @@
78
83
  }
79
84
  },
80
85
  "peerDependencies": {
81
- "@jskit-ai/kernel": "0.2.11"
86
+ "@jskit-ai/kernel": "0.2.12"
82
87
  }
83
88
  }
@@ -1,8 +1,13 @@
1
+ import diagnosticsChannel from "node:diagnostics_channel";
1
2
  import { normalizeDatabaseClient, toKnexClientId } from "../shared/databaseClient.js";
2
3
  import { resolveKnexConnectionFromEnvironment } from "../shared/databaseConnection.js";
3
4
  import { createTransactionManager } from "../shared/transactionManager.js";
4
5
  import { loadKnexFactory } from "./knexFactory.js";
5
6
 
7
+ // The diagnostics channel that carries each statement the shared Knex sends (Knex's `query` event: its `sql`,
8
+ // `bindings` and method), for observers in the process that booted the runtime, such as countRuntimeQueries().
9
+ const RUNTIME_QUERY_CHANNEL = "jskit:database:query";
10
+
6
11
  /** @param {import("./types.js").RuntimeDatabaseDriver} driver */
7
12
  function resolveDriverDialectId(driver) {
8
13
  const dialectId = normalizeDatabaseClient(driver.DIALECT_ID, { allowEmpty: true });
@@ -34,6 +39,8 @@ function createDatabase({ driver, env }) {
34
39
  context: "database runtime"
35
40
  })
36
41
  });
42
+ const statements = diagnosticsChannel.channel(RUNTIME_QUERY_CHANNEL);
43
+ knex.on("query", (query) => statements.publish(query));
37
44
  const transactionManager = createTransactionManager({ knex });
38
45
 
39
46
  return Object.freeze({
@@ -43,4 +50,4 @@ function createDatabase({ driver, env }) {
43
50
  });
44
51
  }
45
52
 
46
- export { createDatabase };
53
+ export { RUNTIME_QUERY_CHANNEL, createDatabase };
@@ -0,0 +1,74 @@
1
+ // Preloaded (`--import`) by `jskit test --databases` into every test file's process. When the run has disposable
2
+ // databases (JSKIT_TEST_DATABASES names the run's directory, from testDatabaseRun() in testDatabases.js), the file takes
3
+ // one no other file is using, for as long as it runs, as both DB_NAME and TEST_DB_NAME, with the connection it was
4
+ // restored on (DB_HOST, DB_PORT, DB_USER, DB_PASSWORD): it never reaches the developer's database. A database a file
5
+ // left with other tables or another migration ledger (a migration test) is restored from the snapshot before the next
6
+ // file takes it.
7
+ //
8
+ // The directory holds the run's settings (only this user may read them: they include the connection password, which
9
+ // reaches only the test files' own processes) and a lock file per database holding its user's process id. The runner's own `node --test` process and every worker
10
+ // thread a test starts load this module too, since they inherit --import: there it does nothing (and loads nothing
11
+ // more), and nothing a file starts takes a database of its own.
12
+ import { closeSync, openSync, readFileSync, rmSync, writeSync } from "node:fs";
13
+ import path from "node:path";
14
+ import process from "node:process";
15
+ import { isMainThread } from "node:worker_threads";
16
+
17
+ const CLAIM_TIMEOUT_MS = 60_000;
18
+
19
+ function isRunning(pid) {
20
+ try {
21
+ process.kill(pid, 0);
22
+ return true;
23
+ } catch (error) {
24
+ return error.code === "EPERM";
25
+ }
26
+ }
27
+
28
+ /** Takes the first free database; a lock left by a process that died without releasing it is taken over. */
29
+ async function claimDatabase(directory, databases) {
30
+ const deadline = Date.now() + CLAIM_TIMEOUT_MS;
31
+ while (Date.now() < deadline) {
32
+ for (const [index, database] of databases.entries()) {
33
+ const lock = path.join(directory, `${index}.lock`);
34
+ try {
35
+ const descriptor = openSync(lock, "wx");
36
+ writeSync(descriptor, String(process.pid));
37
+ closeSync(descriptor);
38
+ process.once("exit", () => rmSync(lock, { force: true }));
39
+ return database;
40
+ } catch (error) {
41
+ if (error.code !== "EEXIST") throw error;
42
+ }
43
+ let owner = 0;
44
+ try {
45
+ owner = Number.parseInt(readFileSync(lock, "utf8"), 10);
46
+ } catch (error) {
47
+ if (error.code !== "ENOENT") throw error;
48
+ }
49
+ if (owner && !isRunning(owner)) rmSync(lock, { force: true });
50
+ }
51
+ await new Promise((resolve) => setTimeout(resolve, 25));
52
+ }
53
+ throw new Error(`No test database came free within ${CLAIM_TIMEOUT_MS / 1000}s.`);
54
+ }
55
+
56
+ const directory = process.env.JSKIT_TEST_DATABASES;
57
+ if (isMainThread && process.env.NODE_TEST_CONTEXT && directory) {
58
+ delete process.env.JSKIT_TEST_DATABASES;
59
+ const { RUN_SETTINGS_FILE, matchesSnapshot, restoreTestDatabase } = await import("./testDatabases.js");
60
+ const { databases, snapshot, knexConfig } = JSON.parse(readFileSync(path.join(directory, RUN_SETTINGS_FILE), "utf8"));
61
+ const database = await claimDatabase(directory, databases);
62
+ if (!(await matchesSnapshot(knexConfig, database, snapshot))) {
63
+ await restoreTestDatabase(knexConfig, database, snapshot);
64
+ }
65
+ const { host, port, user, password } = knexConfig.connection;
66
+ Object.assign(process.env, {
67
+ DB_HOST: String(host),
68
+ DB_PORT: String(port),
69
+ DB_USER: String(user),
70
+ DB_PASSWORD: String(password ?? ""),
71
+ DB_NAME: database,
72
+ TEST_DB_NAME: database
73
+ });
74
+ }
@@ -0,0 +1,585 @@
1
+ // The disposable databases of a test run, and the stored schema snapshots they are restored from.
2
+ //
3
+ // A run gets one database per test file or browser worker running at once: the designated one (TEST_DB_NAME for Node
4
+ // test files, BROWSER_TEST_DB_NAME for browser workers), then copies named after it (`<name>_1`, `_2`, ...) where the
5
+ // database user may create them. Each starts from a snapshot of the schema the migrations build, with the rows they
6
+ // write, instead of replaying every migration. A snapshot is taken once from a real replay into the designated
7
+ // database, keyed by the dialect, the names and contents of the migrations and the application's package-lock.json
8
+ // (a package upgrade can change what its migrations' helpers build), so changing, adding or reordering a migration, or
9
+ // installing other packages, takes a new one. An application whose schema does not come from its migrations alone, such
10
+ // as one whose migrations start from a baseline schema file, gives its own `schema.build` and the files it reads, which
11
+ // join the key:
12
+ //
13
+ // - MySQL and MariaDB: each table's SHOW CREATE TABLE and its rows, in a file under
14
+ // `node_modules/.cache/jskit-test-databases/` named after the key and the server's version;
15
+ // - PostgreSQL: a prepared database, `<name>_snapshot`, that every copy is created from with
16
+ // CREATE DATABASE ... TEMPLATE (STRATEGY FILE_COPY from PostgreSQL 15).
17
+ //
18
+ // The database the application connects to (DB_NAME, the one DATABASE_URL names, the Knex configuration's own) and
19
+ // PROD_DB_NAME are never touched. Two runs at once need different database names, as before.
20
+ //
21
+ // A database test that guards a path's cost counts the statements the path sends with countQueries(), or, for a route
22
+ // of a server the test booted, countRuntimeQueries().
23
+ import { createHash } from "node:crypto";
24
+ import diagnosticsChannel from "node:diagnostics_channel";
25
+ import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
26
+ import os from "node:os";
27
+ import path from "node:path";
28
+ import { fileURLToPath } from "node:url";
29
+ import { parseDatabaseUrl } from "../shared/databaseConnection.js";
30
+ import { RUNTIME_QUERY_CHANNEL } from "./database.js";
31
+ import { loadKnexFactory } from "./knexFactory.js";
32
+
33
+ const SNAPSHOT_DIRECTORY = path.join("node_modules", ".cache", "jskit-test-databases");
34
+ // Room for a `_snapshot` or copy suffix within MySQL's 64 and PostgreSQL's 63 characters.
35
+ const DISPOSABLE_NAME = /^[A-Za-z0-9_]{1,48}$/u;
36
+ const TEST_DATABASE_VARIABLES = Object.freeze(["TEST_DB_NAME", "BROWSER_TEST_DB_NAME"]);
37
+ const MYSQL_ACCESS_DENIED = new Set(["ER_DBACCESS_DENIED_ERROR", "ER_ACCESS_DENIED_ERROR", "ER_SPECIFIC_ACCESS_DENIED_ERROR"]);
38
+ // A database or table the user may not use (not a wrong password, which fails the run).
39
+ const MYSQL_REFUSED_PRIVILEGE = new Set(["ER_DBACCESS_DENIED_ERROR", "ER_SPECIFIC_ACCESS_DENIED_ERROR", "ER_TABLEACCESS_DENIED_ERROR"]);
40
+ const POSTGRES_INSUFFICIENT_PRIVILEGE = "42501";
41
+ // The run's settings, connection password included, in a file only this user can read; the environment names its
42
+ // directory only.
43
+ const RUN_SETTINGS_FILE = "settings.json";
44
+ // The database a PostgreSQL connection uses to create, drop and copy the test databases.
45
+ const POSTGRES_MAINTENANCE_DATABASE = "postgres";
46
+ // PostgreSQL 15 and later copy a template file by file (STRATEGY FILE_COPY) instead of through the write-ahead log, the
47
+ // default there: about half the time per copy of a test schema, and less again with copies made at once.
48
+ const POSTGRES_FILE_COPY_VERSION = 150000;
49
+ // MySQL values read as text, so every captured row writes back as the SQL literal it came from.
50
+ const MYSQL_SNAPSHOT_CONNECTION = Object.freeze({
51
+ dateStrings: true,
52
+ jsonStrings: true,
53
+ supportBigNumbers: true,
54
+ bigNumberStrings: true
55
+ });
56
+ const MYSQL_GENERATED_COLUMN = /\b(?:VIRTUAL|STORED|PERSISTENT) GENERATED\b/iu;
57
+ const INSERT_BATCH = 500;
58
+ const TEST_DATABASE_FILE = fileURLToPath(new URL("./testDatabaseFile.js", import.meta.url));
59
+ const LOCKFILE = "package-lock.json";
60
+
61
+ /**
62
+ * @typedef {object} TestDatabaseSchema How a new snapshot's schema is built when the migrations alone do not build it.
63
+ * @property {string[]} [files] Every file the build reads, its own module included, relative to the application root:
64
+ * their contents join the snapshot key, so changing one takes a new snapshot.
65
+ * @property {(knex: import("knex").Knex) => Promise<unknown>} [build] Builds the schema, and any rows every test
66
+ * database starts with, in the empty database `knex` connects to, instead of `knex.migrate.latest()`: for example a
67
+ * baseline schema file, then the migrations.
68
+ */
69
+
70
+ const folded = (value) => String(value ?? "").trim().toLowerCase();
71
+ const milliseconds = (since) => Math.round(performance.now() - since);
72
+ const ledgerOf = (knexConfig) => knexConfig.migrations?.tableName || "knex_migrations";
73
+
74
+ /**
75
+ * The databases for `count` test files or workers at once: `name`, then its numbered copies.
76
+ * @param {string} name
77
+ * @param {number} count
78
+ */
79
+ function testDatabaseNames(name, count) {
80
+ return Array.from({ length: count }, (_, index) => (index ? `${name}_${index}` : name));
81
+ }
82
+
83
+ /** The database the application's environment connects to: DB_NAME, else the one DATABASE_URL names. */
84
+ function connectedDatabaseName(environment) {
85
+ return String(environment.DB_NAME || "").trim() ||
86
+ parseDatabaseUrl(environment.DATABASE_URL, { allowEmpty: true })?.database || "";
87
+ }
88
+
89
+ /**
90
+ * The disposable database `variable` names, checked with its first `count - 1` copies: never production, never the
91
+ * application's database (DB_NAME, DATABASE_URL's, PROD_DB_NAME, and `connected`, the one its Knex configuration
92
+ * names), and never the other test database or its copies.
93
+ * @param {Record<string, string | undefined>} environment
94
+ * @param {"TEST_DB_NAME" | "BROWSER_TEST_DB_NAME"} variable
95
+ * @param {number} [count]
96
+ * @param {{ connected?: string }} [options]
97
+ */
98
+ function requireDisposableDatabase(environment, variable, count = 1, { connected = "" } = {}) {
99
+ if (environment.NODE_ENV === "production") {
100
+ throw new Error("Test databases are never prepared with NODE_ENV=production.");
101
+ }
102
+ const name = String(environment[variable] || "").trim();
103
+ if (!DISPOSABLE_NAME.test(name)) {
104
+ throw new Error(`${variable} must name a disposable test database: letters, digits and _, at most 48 characters.`);
105
+ }
106
+ const application = [connectedDatabaseName(environment), environment.DB_NAME, environment.PROD_DB_NAME, connected]
107
+ .map(folded).filter(Boolean);
108
+ const others = TEST_DATABASE_VARIABLES.filter((other) => other !== variable)
109
+ .map((other) => folded(environment[other])).filter(Boolean);
110
+ // `copy` is `original` or one of its numbered copies, whatever the count of either run.
111
+ const isCopy = (copy, original) => copy === original ||
112
+ (copy.startsWith(`${original}_`) && /^\d+$/u.test(copy.slice(original.length + 1)));
113
+ for (const candidate of testDatabaseNames(name, count).map(folded)) {
114
+ if (application.includes(candidate)) {
115
+ throw new Error(`${variable} and its copies must differ from the application's DB_NAME, DATABASE_URL and PROD_DB_NAME (${candidate}).`);
116
+ }
117
+ }
118
+ for (const other of others) {
119
+ if (isCopy(folded(name), other) || isCopy(other, folded(name))) {
120
+ throw new Error(`${variable} and its copies must differ from the other test database and its copies (${other}).`);
121
+ }
122
+ }
123
+ return name;
124
+ }
125
+
126
+ function knexOn(knexConfig, database, connectionOptions = {}) {
127
+ return loadKnexFactory()({
128
+ ...knexConfig,
129
+ connection: { ...knexConfig.connection, ...connectionOptions, database },
130
+ pool: { min: 0, max: 1 }
131
+ });
132
+ }
133
+
134
+ async function withKnex(knexConfig, database, use, connectionOptions) {
135
+ const knex = knexOn(knexConfig, database, connectionOptions);
136
+ try {
137
+ return await use(knex);
138
+ } finally {
139
+ await knex.destroy();
140
+ }
141
+ }
142
+
143
+ /**
144
+ * Every migration Knex applies, in its order, and the key of the schema they build: a digest of the dialect, of each
145
+ * migration's name and contents, of each of `schemaFiles` (a schema build's own files) and of the application's
146
+ * package-lock.json. A migration's own file does not show a change in a helper it imports from an installed package;
147
+ * the lockfile does, when that package is upgraded.
148
+ * @param {{ client: string, connection: object, migrations?: object }} knexConfig
149
+ * @param {{ appRoot?: string, schemaFiles?: string[] }} [options] `schemaFiles` are relative to `appRoot`.
150
+ * @returns {Promise<{ key: string, migrations: string[] }>}
151
+ */
152
+ async function testDatabaseSnapshotKey(knexConfig, { appRoot = process.cwd(), schemaFiles = [] } = {}) {
153
+ // Without a connection, Knex lists the migrations without loading the database driver or connecting.
154
+ const knex = loadKnexFactory()({ client: knexConfig.client, migrations: knexConfig.migrations });
155
+ try {
156
+ const { migrationSource: source, loadExtensions } = knex.migrate.config;
157
+ const hash = createHash("sha256").update(`${knexConfig.client}\n`);
158
+ const migrations = [];
159
+ for (const migration of await source.getMigrations(loadExtensions)) {
160
+ const name = source.getMigrationName(migration);
161
+ migrations.push(name);
162
+ hash.update(`${name}\0`);
163
+ hash.update(readFileSync(path.join(migration.directory, migration.file)));
164
+ }
165
+ for (const file of schemaFiles) {
166
+ hash.update(`\0${file}\0`);
167
+ hash.update(readFileSync(path.resolve(appRoot, file)));
168
+ }
169
+ const lockfile = path.join(path.resolve(appRoot), LOCKFILE);
170
+ hash.update(`\0${LOCKFILE}\0`);
171
+ if (existsSync(lockfile)) hash.update(readFileSync(lockfile));
172
+ return { key: hash.digest("hex").slice(0, 32), migrations };
173
+ } finally {
174
+ await knex.destroy();
175
+ }
176
+ }
177
+
178
+ async function ledgerNames(knex, ledger, tables) {
179
+ if (!tables.includes(ledger)) return [];
180
+ const result = await knex.raw("SELECT name FROM ?? ORDER BY id", [ledger]);
181
+ return (knex.client.config.client === "pg" ? result.rows : result[0]).map(({ name }) => name);
182
+ }
183
+
184
+ const mysql = {
185
+ async tables(knex, database) {
186
+ const [rows] = await knex.raw(
187
+ "SELECT TABLE_NAME AS name, TABLE_TYPE AS type FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?", [database]
188
+ );
189
+ if (rows.some(({ type }) => type !== "BASE TABLE")) throw new Error(`${database} holds a view; test databases hold tables only.`);
190
+ return rows.map(({ name }) => name).sort();
191
+ },
192
+
193
+ // Runs on the pool's one connection, so FOREIGN_KEY_CHECKS applies to every statement.
194
+ async replaceTables(knex, database, statements) {
195
+ const [[actual]] = await knex.raw("SELECT DATABASE() AS name");
196
+ if (actual.name !== database) throw new Error(`A test database restore requires a connection to ${database}.`);
197
+ const existing = await mysql.tables(knex, database);
198
+ await knex.raw("SET FOREIGN_KEY_CHECKS = 0");
199
+ try {
200
+ for (const name of existing) await knex.raw("DROP TABLE ??", [name]);
201
+ for (const statement of statements) await knex.raw(statement);
202
+ } finally {
203
+ await knex.raw("SET FOREIGN_KEY_CHECKS = 1");
204
+ }
205
+ },
206
+
207
+ /** Each table's definition and its rows as INSERT statements (generated columns left to the server). */
208
+ async capture(knex, database) {
209
+ // SHOW CREATE TABLE carries neither, so a copy would silently lack them.
210
+ const [[{ triggers, routines, events }]] = await knex.raw(
211
+ "SELECT (SELECT COUNT(*) FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = ?) AS triggers, " +
212
+ "(SELECT COUNT(*) FROM information_schema.ROUTINES WHERE ROUTINE_SCHEMA = ?) AS routines, " +
213
+ "(SELECT COUNT(*) FROM information_schema.EVENTS WHERE EVENT_SCHEMA = ?) AS events",
214
+ [database, database, database]
215
+ );
216
+ if (Number(triggers) + Number(routines) + Number(events) > 0) {
217
+ throw new Error(`${database} holds triggers, routines or events; MySQL test databases restored from a snapshot hold tables only.`);
218
+ }
219
+ const [columns] = await knex.raw(
220
+ "SELECT TABLE_NAME AS tableName, COLUMN_NAME AS name, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ?",
221
+ [database]
222
+ );
223
+ const tables = [];
224
+ for (const name of await mysql.tables(knex, database)) {
225
+ const [[definition]] = await knex.raw("SHOW CREATE TABLE ??", [name]);
226
+ const [rows] = await knex.raw("SELECT * FROM ??", [name]);
227
+ const generated = columns.filter((column) => column.tableName === name && MYSQL_GENERATED_COLUMN.test(column.extra))
228
+ .map((column) => column.name);
229
+ for (const row of rows) for (const column of generated) delete row[column];
230
+ const inserts = [];
231
+ for (let start = 0; start < rows.length; start += INSERT_BATCH) {
232
+ inserts.push(knex(name).insert(rows.slice(start, start + INSERT_BATCH)).toQuery());
233
+ }
234
+ tables.push({ name, create: definition["Create Table"], inserts });
235
+ }
236
+ return tables;
237
+ },
238
+
239
+ async ensureSnapshot(knexConfig, database, key, appRoot, build) {
240
+ // Another server version may write, or accept, another SHOW CREATE TABLE.
241
+ const version = await withKnex(knexConfig, undefined, async (knex) => (await knex.raw("SELECT VERSION() AS version"))[0][0].version);
242
+ const server = createHash("sha256").update(String(version)).digest("hex").slice(0, 8);
243
+ const file = path.join(appRoot, SNAPSHOT_DIRECTORY, `${key}-${server}.json`);
244
+ if (existsSync(file)) {
245
+ const { migrations, tables } = JSON.parse(readFileSync(file, "utf8"));
246
+ return { key, file, migrations, tables: tables.map(({ name }) => name), replayed: false };
247
+ }
248
+ await mysql.createDatabase(knexConfig, database);
249
+ // The schema is built on the application's own connection settings; only the capture reads values as text.
250
+ await withKnex(knexConfig, database, async (knex) => {
251
+ await mysql.replaceTables(knex, database, []);
252
+ await build(knex);
253
+ });
254
+ const { migrations } = await mysql.outline(knexConfig, database);
255
+ const tables = await withKnex(knexConfig, database, (knex) => mysql.capture(knex, database), MYSQL_SNAPSHOT_CONNECTION);
256
+ mkdirSync(path.dirname(file), { recursive: true });
257
+ // Written whole, then renamed, so a run beside this one never reads half a snapshot.
258
+ writeFileSync(`${file}.${process.pid}`, JSON.stringify({ migrations, tables }));
259
+ renameSync(`${file}.${process.pid}`, file);
260
+ return { key, file, migrations, tables: tables.map(({ name }) => name), replayed: true };
261
+ },
262
+
263
+ /** Creates `database` unless it exists; false when the database user may not create it. */
264
+ async createDatabase(knexConfig, database) {
265
+ return withKnex(knexConfig, undefined, async (knex) => {
266
+ try {
267
+ await knex.raw("CREATE DATABASE IF NOT EXISTS ?? CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci", [database]);
268
+ return true;
269
+ } catch (error) {
270
+ if (MYSQL_ACCESS_DENIED.has(error.code)) return false;
271
+ throw error;
272
+ }
273
+ });
274
+ },
275
+
276
+ async restore(knexConfig, database, snapshot) {
277
+ const { tables } = JSON.parse(readFileSync(snapshot.file, "utf8"));
278
+ // A test may have dropped the whole database; one the user may not create but that exists is restored in place.
279
+ await mysql.createDatabase(knexConfig, database);
280
+ await withKnex(knexConfig, database, (knex) => mysql.replaceTables(knex, database, [
281
+ ...tables.map(({ create }) => create),
282
+ ...tables.flatMap(({ inserts }) => inserts)
283
+ ]));
284
+ return true;
285
+ },
286
+
287
+ async provide(knexConfig, databases, snapshot) {
288
+ const provided = [];
289
+ for (const [index, database] of databases.entries()) {
290
+ // The designated database may exist already for a user who cannot create databases.
291
+ if (!(await mysql.createDatabase(knexConfig, database)) && index > 0) break;
292
+ provided.push(database);
293
+ }
294
+ await Promise.all(provided.map((database) => mysql.restore(knexConfig, database, snapshot)));
295
+ return provided;
296
+ },
297
+
298
+ async outline(knexConfig, database) {
299
+ return withKnex(knexConfig, database, async (knex) => {
300
+ const tables = await mysql.tables(knex, database);
301
+ return { tables, migrations: await ledgerNames(knex, ledgerOf(knexConfig), tables) };
302
+ });
303
+ }
304
+ };
305
+
306
+ const postgres = {
307
+ // Tables and views alike: a copy made from the template database carries everything the migrations made.
308
+ async tables(knex) {
309
+ const { rows } = await knex.raw("SELECT table_name AS name FROM information_schema.tables WHERE table_schema = current_schema()");
310
+ return rows.map(({ name }) => name).sort();
311
+ },
312
+
313
+ maintenance(knexConfig, use) {
314
+ return withKnex(knexConfig, POSTGRES_MAINTENANCE_DATABASE, use);
315
+ },
316
+
317
+ async ensureSnapshot(knexConfig, database, key, _appRoot, build) {
318
+ const snapshotDatabase = `${database}_snapshot`;
319
+ const label = `jskit-test-snapshot ${key}`;
320
+ const current = await postgres.maintenance(knexConfig, async (knex) => {
321
+ const { rows } = await knex.raw(
322
+ "SELECT shobj_description(oid, 'pg_database') AS label FROM pg_database WHERE datname = ?", [snapshotDatabase]
323
+ );
324
+ return rows[0]?.label || "";
325
+ });
326
+ const replayed = current !== label;
327
+ if (replayed) {
328
+ await postgres.maintenance(knexConfig, async (knex) => {
329
+ await knex.raw("DROP DATABASE IF EXISTS ??", [snapshotDatabase]);
330
+ await knex.raw("CREATE DATABASE ??", [snapshotDatabase]);
331
+ });
332
+ await withKnex(knexConfig, snapshotDatabase, build);
333
+ // The key is hexadecimal, so it is safe inline; COMMENT takes no bound parameters.
334
+ await postgres.maintenance(knexConfig, (knex) => knex.raw(`COMMENT ON DATABASE ?? IS '${label}'`, [snapshotDatabase]));
335
+ }
336
+ // Read, then closed: a template database is copied only while nothing is connected to it.
337
+ const { tables, migrations } = await postgres.outline(knexConfig, snapshotDatabase);
338
+ return { key, database: snapshotDatabase, migrations, tables, replayed };
339
+ },
340
+
341
+ /** Recreates `database` from the snapshot; false when the database user may not create databases. */
342
+ async restore(knexConfig, database, snapshot) {
343
+ return postgres.maintenance(knexConfig, async (knex) => {
344
+ try {
345
+ await knex.raw("DROP DATABASE IF EXISTS ??", [database]);
346
+ const { rows: [{ server_version_num: version }] } = await knex.raw("SHOW server_version_num");
347
+ const strategy = Number(version) >= POSTGRES_FILE_COPY_VERSION ? " STRATEGY FILE_COPY" : "";
348
+ await knex.raw(`CREATE DATABASE ?? TEMPLATE ??${strategy}`, [database, snapshot.database]);
349
+ return true;
350
+ } catch (error) {
351
+ if (error.code === POSTGRES_INSUFFICIENT_PRIVILEGE) return false;
352
+ throw error;
353
+ }
354
+ });
355
+ },
356
+
357
+ // One at a time: each copy reads the template while it is made.
358
+ async provide(knexConfig, databases, snapshot) {
359
+ const provided = [];
360
+ for (const database of databases) {
361
+ if (!(await postgres.restore(knexConfig, database, snapshot))) break;
362
+ provided.push(database);
363
+ }
364
+ if (provided.length < 1) throw new Error(`The database user may not create ${databases[0]} from the snapshot (CREATEDB).`);
365
+ return provided;
366
+ },
367
+
368
+ async outline(knexConfig, database) {
369
+ return withKnex(knexConfig, database, async (knex) => {
370
+ const tables = await postgres.tables(knex);
371
+ return { tables, migrations: await ledgerNames(knex, ledgerOf(knexConfig), tables) };
372
+ });
373
+ }
374
+ };
375
+
376
+ const DIALECTS = Object.freeze({
377
+ mysql2: mysql,
378
+ pg: postgres
379
+ });
380
+
381
+ function dialectOf(knexConfig) {
382
+ const dialect = DIALECTS[knexConfig?.client];
383
+ if (!dialect || !knexConfig.connection || typeof knexConfig.connection !== "object") {
384
+ throw new Error("Test databases need the application's resolved MySQL (mysql2) or PostgreSQL (pg) Knex configuration.");
385
+ }
386
+ return dialect;
387
+ }
388
+
389
+ /**
390
+ * Prepares the disposable databases for up to `count` test files or browser workers at once, each holding the current
391
+ * migrations' schema afresh, then the seed's rows when given: the database `variable` names, then the numbered copies
392
+ * the database user may create.
393
+ * @param {object} options
394
+ * @param {{ client: string, connection: object, migrations?: object }} options.knexConfig The application's Knex
395
+ * migration configuration (its knexfile.js), from `createKnexMigrationConfigFromApp()`.
396
+ * @param {Record<string, string | undefined>} options.environment The application's environment, such as `process.env`.
397
+ * @param {"TEST_DB_NAME" | "BROWSER_TEST_DB_NAME"} [options.variable]
398
+ * @param {number} [options.count]
399
+ * @param {(transaction: import("knex").Knex.Transaction) => unknown} [options.seed] Runs in each database, in one
400
+ * transaction, after its restore.
401
+ * @param {TestDatabaseSchema} [options.schema] How a new snapshot's schema is built, when not by the migrations alone.
402
+ * @param {string} [options.appRoot]
403
+ */
404
+ async function prepareTestDatabases({
405
+ knexConfig,
406
+ environment,
407
+ variable = "TEST_DB_NAME",
408
+ count = 1,
409
+ seed,
410
+ schema = {},
411
+ appRoot = process.cwd()
412
+ }) {
413
+ const started = performance.now();
414
+ const dialect = dialectOf(knexConfig);
415
+ const name = requireDisposableDatabase(environment, variable, count, { connected: knexConfig.connection.database });
416
+ const root = path.resolve(appRoot);
417
+ const { key, migrations } = await testDatabaseSnapshotKey(knexConfig, { appRoot: root, schemaFiles: schema.files });
418
+ const build = schema.build ?? ((knex) => knex.migrate.latest());
419
+ const snapshot = await dialect.ensureSnapshot(knexConfig, name, key, root, build);
420
+ const restoreStarted = performance.now();
421
+ const databases = await dialect.provide(knexConfig, testDatabaseNames(name, count), snapshot);
422
+ const restoreMs = milliseconds(restoreStarted);
423
+ const seedStarted = performance.now();
424
+ if (seed) {
425
+ await Promise.all(databases.map((database) => withKnex(knexConfig, database, (knex) => knex.transaction(seed))));
426
+ }
427
+ const seedMs = milliseconds(seedStarted);
428
+ const { replayed, ...stored } = snapshot;
429
+ return {
430
+ databases,
431
+ snapshot: stored,
432
+ summary: `${databases.length} test database(s) from ${name} ready in ${milliseconds(started)}ms: ` +
433
+ (replayed && schema.build ? "schema built once for a new snapshot, " : "") +
434
+ (replayed && !schema.build ? `${migrations.length} migrations replayed once for a new snapshot, ` : "") +
435
+ (replayed ? "" : "stored snapshot, ") +
436
+ `restore ${restoreMs}ms${seed ? `, seed ${seedMs}ms` : ""}`
437
+ };
438
+ }
439
+
440
+ /**
441
+ * Whether `database` still holds the snapshot's tables and migration ledger, which a test that migrates up or down,
442
+ * or drops tables, changes.
443
+ */
444
+ async function matchesSnapshot(knexConfig, database, snapshot) {
445
+ let outline;
446
+ try {
447
+ outline = await dialectOf(knexConfig).outline(knexConfig, database);
448
+ } catch (error) {
449
+ // A test dropped the whole database (MySQL ER_BAD_DB_ERROR, PostgreSQL 3D000): the restore recreates it.
450
+ if (error.code === "ER_BAD_DB_ERROR" || error.code === "3D000") return false;
451
+ throw error;
452
+ }
453
+ const { tables, migrations } = outline;
454
+ return tables.join("\n") === snapshot.tables.join("\n") && migrations.join("\n") === snapshot.migrations.join("\n");
455
+ }
456
+
457
+ /** Replaces every table of `database`, which must be disposable, with the snapshot's. */
458
+ async function restoreTestDatabase(knexConfig, database, snapshot) {
459
+ if (!(await dialectOf(knexConfig).restore(knexConfig, database, snapshot))) {
460
+ throw new Error(`The database user may not recreate ${database}.`);
461
+ }
462
+ }
463
+
464
+ /** The server refused the database user a database or table it needs, rather than failing in some other way. */
465
+ function refusedPrivilege(error) {
466
+ return MYSQL_REFUSED_PRIVILEGE.has(error?.code) || error?.code === POSTGRES_INSUFFICIENT_PRIVILEGE;
467
+ }
468
+
469
+ /**
470
+ * How `jskit test --databases` runs the Node test files: in parallel, each file with a database of its own for as long
471
+ * as it runs (`testDatabaseFile.js`, preloaded into every file's process); or, with no usable TEST_DB_NAME (unset, the
472
+ * connected database itself, or one the database user may not prepare), one file at a time on the connected database,
473
+ * as before. Anything else that stops the preparation, such as a TEST_DB_NAME copy named like a protected database or
474
+ * an unreachable server, fails the run.
475
+ * @param {object} options
476
+ * @param {{ client: string, connection: object, migrations?: object }} options.knexConfig
477
+ * @param {Record<string, string | undefined>} options.environment
478
+ * @param {number} options.count How many test files run at once: at most the run's test budget.
479
+ * @param {TestDatabaseSchema} [options.schema]
480
+ * @param {string} [options.appRoot]
481
+ */
482
+ async function testDatabaseRun({ knexConfig, environment, count, schema, appRoot = process.cwd() }) {
483
+ const oneAtATime = (reason) => ({
484
+ concurrency: 1,
485
+ env: {},
486
+ preload: "",
487
+ summary: `test files run one at a time on the connected database: ${reason}`,
488
+ cleanup() {}
489
+ });
490
+ const testDatabase = String(environment.TEST_DB_NAME || "").trim();
491
+ if (!testDatabase) return oneAtATime("TEST_DB_NAME is not set");
492
+ if (folded(connectedDatabaseName(environment)) === folded(testDatabase)) {
493
+ return oneAtATime(`the connected database is TEST_DB_NAME (${testDatabase})`);
494
+ }
495
+ let prepared;
496
+ try {
497
+ prepared = await prepareTestDatabases({ knexConfig, environment, count, schema, appRoot });
498
+ } catch (error) {
499
+ if (!refusedPrivilege(error)) throw error;
500
+ return oneAtATime(`the database user may not prepare ${testDatabase} (${error.code})`);
501
+ }
502
+ // mkdtemp makes the directory for this user alone; the settings file holds the connection password.
503
+ const directory = mkdtempSync(path.join(os.tmpdir(), "jskit-test-databases-"));
504
+ const { client, connection, migrations } = knexConfig;
505
+ const settings = path.join(directory, RUN_SETTINGS_FILE);
506
+ writeFileSync(settings, JSON.stringify({
507
+ databases: prepared.databases,
508
+ snapshot: prepared.snapshot,
509
+ knexConfig: { client, connection, migrations: { tableName: ledgerOf({ migrations }) } }
510
+ }), { mode: 0o600 });
511
+ return {
512
+ concurrency: prepared.databases.length,
513
+ env: { JSKIT_TEST_DATABASES: directory },
514
+ preload: TEST_DATABASE_FILE,
515
+ summary: prepared.summary,
516
+ cleanup: () => rmSync(directory, { recursive: true, force: true })
517
+ };
518
+ }
519
+
520
+ /**
521
+ * Runs `work` while `listen` hands each statement to the recorder it is given, and stops listening when `work` ends.
522
+ * @template T
523
+ * @param {(record: (query: { sql: string }) => void) => () => void} listen Starts listening; returns how to stop.
524
+ * @param {() => Promise<T> | T} work
525
+ * @returns {Promise<{ result: T, count: number, queries: string[] }>}
526
+ */
527
+ async function recordStatements(listen, work) {
528
+ const queries = [];
529
+ const stop = listen((query) => {
530
+ queries.push(String(query.sql));
531
+ });
532
+ try {
533
+ const result = await work();
534
+ return { result, count: queries.length, queries };
535
+ } finally {
536
+ stop();
537
+ }
538
+ }
539
+
540
+ /**
541
+ * Runs `work` and records the SQL of every statement `knex` sends meanwhile, its transactions' included (their BEGIN
542
+ * and COMMIT among them), for a test that compares a path's statements at a few rows and at many. Pass the Knex
543
+ * instance or transaction the work runs on; statements anything else sends through it meanwhile count too.
544
+ * @template T
545
+ * @param {import("knex").Knex | import("knex").Knex.Transaction} knex
546
+ * @param {() => Promise<T> | T} work
547
+ * @returns {Promise<{ result: T, count: number, queries: string[] }>} `work`'s result, how many statements it sent and
548
+ * their SQL, in order.
549
+ */
550
+ async function countQueries(knex, work) {
551
+ return recordStatements((record) => {
552
+ knex.on("query", record);
553
+ return () => knex.off("query", record);
554
+ }, work);
555
+ }
556
+
557
+ /**
558
+ * Runs `work` and records the SQL of every statement the application runtime's own database sends meanwhile: the
559
+ * `runtime.database` Knex of each server this process booted, as a test that creates the server with `createServer()`
560
+ * and calls a route with `app.inject()` does, its transactions' included. A Knex the test made itself is not counted,
561
+ * and neither is a server in another process. Statements the runtime sends for anything else meanwhile count too.
562
+ * @template T
563
+ * @param {() => Promise<T> | T} work
564
+ * @returns {Promise<{ result: T, count: number, queries: string[] }>} `work`'s result, how many statements the runtime
565
+ * sent and their SQL, in order.
566
+ */
567
+ async function countRuntimeQueries(work) {
568
+ return recordStatements((record) => {
569
+ diagnosticsChannel.subscribe(RUNTIME_QUERY_CHANNEL, record);
570
+ return () => diagnosticsChannel.unsubscribe(RUNTIME_QUERY_CHANNEL, record);
571
+ }, work);
572
+ }
573
+
574
+ export {
575
+ RUN_SETTINGS_FILE,
576
+ countQueries,
577
+ countRuntimeQueries,
578
+ matchesSnapshot,
579
+ prepareTestDatabases,
580
+ requireDisposableDatabase,
581
+ restoreTestDatabase,
582
+ testDatabaseNames,
583
+ testDatabaseRun,
584
+ testDatabaseSnapshotKey
585
+ };
@@ -6,6 +6,7 @@ import test from "node:test";
6
6
 
7
7
  import { defineProvider, createCapabilityRuntime } from "@jskit-ai/kernel/shared/capabilities";
8
8
  import { DatabaseProvider } from "../src/server/providers/DatabaseProvider.js";
9
+ import { countRuntimeQueries } from "../src/server/testDatabases.js";
9
10
 
10
11
  function driverProvider(dialectId = "mysql2") {
11
12
  return defineProvider({
@@ -51,8 +52,11 @@ async function withAppRootKnexStub(callback) {
51
52
  " __config: config,",
52
53
  " destroyCalls: 0,",
53
54
  " rawQueries: [],",
55
+ " queryListeners: [],",
56
+ " on: function on(event, listener) { if (event === 'query') this.queryListeners.push(listener); return this; },",
54
57
  " raw: async function raw(sql) {",
55
58
  " this.rawQueries.push(sql);",
59
+ " for (const listener of this.queryListeners) listener({ sql });",
56
60
  " if (config.connection.host === 'db.down') throw new Error('connect ECONNREFUSED');",
57
61
  " return [];",
58
62
  " },",
@@ -202,3 +206,24 @@ test("DatabaseProvider starts without health checks", async () => {
202
206
  await runtime.shutdown();
203
207
  });
204
208
  });
209
+
210
+ test("the runtime's statements reach countRuntimeQueries() in the process that booted it, only while the work runs", async () => {
211
+ await withAppRootKnexStub(async () => {
212
+ let database;
213
+ const runtime = createCapabilityRuntime({
214
+ inputs: { "runtime.env": { DB_HOST: "db.local", DB_NAME: "appdb", DB_USER: "appuser" } },
215
+ providers: [driverProvider(), DatabaseProvider, databaseConsumer((value) => { database = value; })]
216
+ });
217
+ await runtime.start();
218
+ await database.knex.raw("select 'before'");
219
+ const counted = await countRuntimeQueries(async () => {
220
+ for (const id of [1, 2, 3]) await database.knex.raw(`select ${id}`);
221
+ return "listed";
222
+ });
223
+ await database.knex.raw("select 'after'");
224
+ assert.deepEqual(counted, { result: "listed", count: 3, queries: ["select 1", "select 2", "select 3"] });
225
+ await assert.rejects(countRuntimeQueries(() => Promise.reject(new Error("stopped"))), /stopped/u);
226
+ assert.equal((await countRuntimeQueries(() => "nothing sent")).count, 0);
227
+ await runtime.shutdown();
228
+ });
229
+ });
@@ -0,0 +1,193 @@
1
+ import assert from "node:assert/strict";
2
+ import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
3
+ import os from "node:os";
4
+ import path from "node:path";
5
+ import test from "node:test";
6
+ import Fastify from "fastify";
7
+ import knexFactory from "knex";
8
+ import { createCapabilityRuntime, defineProvider } from "@jskit-ai/kernel/shared/capabilities";
9
+ import { DatabaseProvider } from "../src/server/providers/DatabaseProvider.js";
10
+ import { createKnexMigrationConfig } from "../src/server/knexMigrationConfig.js";
11
+ import {
12
+ countQueries,
13
+ countRuntimeQueries,
14
+ matchesSnapshot,
15
+ prepareTestDatabases,
16
+ restoreTestDatabase
17
+ } from "../src/server/testDatabases.js";
18
+
19
+ // Runs beside the PostgreSQL test database: run with DB_NAME set to TEST_DB_NAME (and the usual DB_HOST, DB_PORT,
20
+ // DB_USER and DB_PASSWORD), DB_CLIENT unset or pg, and a database user that may create databases (CREATEDB).
21
+ const TEST_DB_NAME = process.env.TEST_DB_NAME?.trim();
22
+ const ON_TEST_DATABASE = Boolean(TEST_DB_NAME) && process.env.DB_NAME === TEST_DB_NAME;
23
+ const POSTGRES_CLIENT = !process.env.DB_CLIENT || process.env.DB_CLIENT === "pg";
24
+ const SKIP = ON_TEST_DATABASE && POSTGRES_CLIENT ? false : "not on a PostgreSQL test database (DB_NAME=$TEST_DB_NAME)";
25
+
26
+ const BOOKS = `exports.up = async (knex) => {
27
+ await knex.schema.createTable("books", (table) => { table.increments("id"); table.string("title"); });
28
+ await knex("books").insert({ title: "Written by the migration" });
29
+ };
30
+ exports.down = () => {};
31
+ `;
32
+
33
+ test("each PostgreSQL test database is a copy of the snapshot database, rebuilt only when a migration changes", {
34
+ skip: SKIP
35
+ }, async (context) => {
36
+ const appRoot = await mkdtemp(path.join(os.tmpdir(), "jskit-test-databases-pg-"));
37
+ await mkdir(path.join(appRoot, "migrations", "constraints"), { recursive: true });
38
+ await writeFile(path.join(appRoot, "migrations", "20200101000000_books.cjs"), BOOKS);
39
+ const name = `${TEST_DB_NAME}_copies`;
40
+ const environment = { ...process.env, DB_CLIENT: "pg", DB_NAME: TEST_DB_NAME, DATABASE_URL: "", TEST_DB_NAME: name };
41
+ const knexConfig = createKnexMigrationConfig({ client: "pg", environment, appRoot });
42
+ const on = (database) => knexFactory({ client: "pg", connection: { ...knexConfig.connection, database } });
43
+ const names = [name, `${name}_1`];
44
+ context.after(async () => {
45
+ const admin = on(TEST_DB_NAME);
46
+ try {
47
+ for (const database of [...names, `${name}_snapshot`]) await admin.raw("DROP DATABASE IF EXISTS ??", [database]);
48
+ } finally {
49
+ await admin.destroy();
50
+ }
51
+ await rm(appRoot, { recursive: true, force: true });
52
+ });
53
+ const seed = (transaction) => transaction("books").insert({ title: "Seeded" });
54
+
55
+ const first = await prepareTestDatabases({ knexConfig, environment, count: 2, seed, appRoot });
56
+ assert.deepEqual(first.databases, names);
57
+ assert.match(first.summary, /1 migrations replayed once for a new snapshot/u);
58
+ assert.equal(first.snapshot.database, `${name}_snapshot`);
59
+ for (const database of names) {
60
+ const db = on(database);
61
+ try {
62
+ assert.deepEqual(await db("books").orderBy("id").pluck("title"), ["Written by the migration", "Seeded"], database);
63
+ } finally {
64
+ await db.destroy();
65
+ }
66
+ }
67
+
68
+ const again = await prepareTestDatabases({ knexConfig, environment, count: 2, appRoot });
69
+ assert.match(again.summary, /stored snapshot/u);
70
+ const copy = on(names[1]);
71
+ try {
72
+ assert.deepEqual(await copy("books").pluck("title"), ["Written by the migration"]);
73
+ await copy.schema.dropTable("books");
74
+ } finally {
75
+ await copy.destroy();
76
+ }
77
+ assert.equal(await matchesSnapshot(knexConfig, names[1], again.snapshot), false);
78
+ await restoreTestDatabase(knexConfig, names[1], again.snapshot);
79
+ assert.equal(await matchesSnapshot(knexConfig, names[1], again.snapshot), true);
80
+
81
+ await writeFile(path.join(appRoot, "migrations", "20200102000000_shelves.cjs"),
82
+ "exports.up = (knex) => knex.schema.createTable(\"shelves\", (table) => table.increments(\"id\"));\nexports.down = () => {};\n");
83
+ const changed = await prepareTestDatabases({ knexConfig, environment, count: 1, appRoot });
84
+ assert.match(changed.summary, /2 migrations replayed once for a new snapshot/u);
85
+ assert.deepEqual(changed.snapshot.tables, ["books", "knex_migrations", "knex_migrations_lock", "shelves"]);
86
+ });
87
+
88
+ test("countQueries() counts a PostgreSQL path's statements, its transactions' included", { skip: SKIP }, async (context) => {
89
+ const { connection } = createKnexMigrationConfig({ client: "pg", environment: { ...process.env, DB_CLIENT: "pg" }, appRoot: os.tmpdir() });
90
+ const db = knexFactory({ client: "pg", connection });
91
+ const table = "query_count_books";
92
+ context.after(async () => {
93
+ await db.schema.dropTableIfExists(table);
94
+ await db.destroy();
95
+ });
96
+ await db.schema.dropTableIfExists(table);
97
+ await db.schema.createTable(table, (columns) => { columns.increments("id"); columns.string("title"); });
98
+ const ids = (await db(table).insert(Array.from({ length: 30 }, (_, index) => ({ title: `Book ${index + 1}` })), ["id"])).map((row) => row.id);
99
+
100
+ const each = await countQueries(db, async () => {
101
+ for (const id of ids) await db(table).where({ id }).first("title");
102
+ });
103
+ assert.equal(each.count, 30);
104
+ assert.equal(each.queries[0], `select "title" from "${table}" where "id" = $1 limit $2`);
105
+ assert.equal((await countQueries(db, () => db(table).whereIn("id", ids).pluck("title"))).count, 1);
106
+ const inTransaction = await countQueries(db, () => db.transaction((trx) => trx(table).count({ books: "*" })));
107
+ assert.deepEqual(inTransaction.queries, ["BEGIN;", `select count(*) as "books" from "${table}"`, "COMMIT;"]);
108
+ });
109
+
110
+ test("countRuntimeQueries() counts a PostgreSQL route's reads through the runtime the test booted", { skip: SKIP }, async (context) => {
111
+ const table = "route_count_books";
112
+ const environment = { ...process.env, DB_CLIENT: "pg" };
113
+ const { connection } = createKnexMigrationConfig({ client: "pg", environment, appRoot: os.tmpdir() });
114
+ const own = knexFactory({ client: "pg", connection });
115
+ const app = Fastify({ logger: false });
116
+ const driver = defineProvider({
117
+ id: "test.database.driver.pg",
118
+ provides: { driver: "runtime.database.driver" },
119
+ setup: () => ({ driver: Object.freeze({ DIALECT_ID: "pg" }) })
120
+ });
121
+ const route = defineProvider({
122
+ id: "test.route",
123
+ requires: { database: "runtime.database" },
124
+ setup({ database }) {
125
+ app.get("/titles/:count", async (request) => {
126
+ const rows = await database.knex(table).select("id").orderBy("id").limit(Number(request.params.count));
127
+ return Promise.all(rows.map(({ id }) => database.knex(table).where({ id }).first("title")));
128
+ });
129
+ return {};
130
+ }
131
+ });
132
+ const runtime = createCapabilityRuntime({ inputs: { "runtime.env": environment }, providers: [driver, DatabaseProvider, route] });
133
+ context.after(async () => {
134
+ await app.close();
135
+ await runtime.shutdown();
136
+ await own.schema.dropTableIfExists(table);
137
+ await own.destroy();
138
+ });
139
+ await own.schema.dropTableIfExists(table);
140
+ await own.schema.createTable(table, (columns) => { columns.increments("id"); columns.string("title"); });
141
+ await own(table).insert(Array.from({ length: 200 }, (_, index) => ({ title: `Title ${index + 1}` })));
142
+ await runtime.start();
143
+
144
+ const counts = [];
145
+ for (const count of [10, 200]) {
146
+ const { result, count: statements } = await countRuntimeQueries(() => app.inject(`/titles/${count}`));
147
+ assert.equal(result.json().length, count);
148
+ counts.push(statements);
149
+ }
150
+ // A read per title: the statements follow the rows.
151
+ assert.deepEqual(counts, [11, 201]);
152
+ });
153
+
154
+ test("a PostgreSQL snapshot built by the application's own schema build carries its baseline and its ledger", {
155
+ skip: SKIP
156
+ }, async (context) => {
157
+ const appRoot = await mkdtemp(path.join(os.tmpdir(), "jskit-test-databases-pg-baseline-"));
158
+ await mkdir(path.join(appRoot, "migrations", "constraints"), { recursive: true });
159
+ await writeFile(path.join(appRoot, "migrations", "20200101000000_shelf_colour.cjs"),
160
+ "exports.up = (knex) => knex.schema.alterTable(\"shelves\", (table) => table.string(\"colour\"));\nexports.down = () => {};\n");
161
+ await writeFile(path.join(appRoot, "baseline.sql"), "CREATE TABLE shelves (id integer PRIMARY KEY, label text)");
162
+ const name = `${TEST_DB_NAME}_baseline`;
163
+ const environment = { ...process.env, DB_CLIENT: "pg", DB_NAME: TEST_DB_NAME, DATABASE_URL: "", TEST_DB_NAME: name };
164
+ const knexConfig = createKnexMigrationConfig({ client: "pg", environment, appRoot });
165
+ context.after(async () => {
166
+ const admin = knexFactory({ client: "pg", connection: knexConfig.connection });
167
+ try {
168
+ for (const database of [name, `${name}_snapshot`]) await admin.raw("DROP DATABASE IF EXISTS ??", [database]);
169
+ } finally {
170
+ await admin.destroy();
171
+ }
172
+ await rm(appRoot, { recursive: true, force: true });
173
+ });
174
+ const schema = {
175
+ files: ["baseline.sql"],
176
+ async build(db) {
177
+ await db.raw(await readFile(path.join(appRoot, "baseline.sql"), "utf8"));
178
+ await db("shelves").insert({ id: 1, label: "Reference" });
179
+ await db.migrate.latest();
180
+ }
181
+ };
182
+
183
+ const prepared = await prepareTestDatabases({ knexConfig, environment, count: 1, schema, appRoot });
184
+ assert.match(prepared.summary, /schema built once for a new snapshot/u);
185
+ assert.deepEqual(prepared.snapshot.migrations, ["20200101000000_shelf_colour.cjs"]);
186
+ const copy = knexFactory({ client: "pg", connection: { ...knexConfig.connection, database: name } });
187
+ try {
188
+ assert.deepEqual(await copy("shelves").select("id", "label", "colour"), [{ id: 1, label: "Reference", colour: null }]);
189
+ } finally {
190
+ await copy.destroy();
191
+ }
192
+ assert.equal(await matchesSnapshot(knexConfig, name, prepared.snapshot), true);
193
+ });
@@ -0,0 +1,158 @@
1
+ import assert from "node:assert/strict";
2
+ import { mkdtemp, mkdir, rename, rm, writeFile } from "node:fs/promises";
3
+ import os from "node:os";
4
+ import path from "node:path";
5
+ import test from "node:test";
6
+ import knexFactory from "knex";
7
+ import { createKnexMigrationConfig } from "../src/server/knexMigrationConfig.js";
8
+ import {
9
+ countQueries,
10
+ requireDisposableDatabase,
11
+ testDatabaseNames,
12
+ testDatabaseRun,
13
+ testDatabaseSnapshotKey
14
+ } from "../src/server/testDatabases.js";
15
+
16
+ const connectionEnvironment = Object.freeze({
17
+ DB_HOST: "127.0.0.1",
18
+ DB_NAME: "catalogue",
19
+ DB_USER: "catalogue_user",
20
+ DB_PASSWORD: "secret"
21
+ });
22
+ const MIGRATION = "exports.up = (knex) => knex.schema.createTable(\"books\", (table) => table.increments());\nexports.down = () => {};\n";
23
+
24
+ test("a run's test databases are the designated one, then its numbered copies", () => {
25
+ assert.deepEqual(testDatabaseNames("catalogue_test", 1), ["catalogue_test"]);
26
+ assert.deepEqual(testDatabaseNames("catalogue_test", 3), ["catalogue_test", "catalogue_test_1", "catalogue_test_2"]);
27
+ });
28
+
29
+ test("a test database and its copies are never the application's database, production or the other test database", () => {
30
+ const environment = { ...connectionEnvironment, TEST_DB_NAME: "catalogue_test", BROWSER_TEST_DB_NAME: "catalogue_browser" };
31
+ assert.equal(requireDisposableDatabase(environment, "TEST_DB_NAME", 4), "catalogue_test");
32
+ assert.equal(requireDisposableDatabase(environment, "BROWSER_TEST_DB_NAME", 4), "catalogue_browser");
33
+ for (const [overrides, variable, count, message] of [
34
+ [{ TEST_DB_NAME: "" }, "TEST_DB_NAME", 1, /must name a disposable test database/u],
35
+ [{ TEST_DB_NAME: "catalogue-test" }, "TEST_DB_NAME", 1, /must name a disposable test database/u],
36
+ [{ TEST_DB_NAME: "Catalogue" }, "TEST_DB_NAME", 1, /DB_NAME, DATABASE_URL and PROD_DB_NAME \(catalogue\)/u],
37
+ [{ DB_NAME: "", DATABASE_URL: "mysql://u:p@db/catalogue_test" }, "TEST_DB_NAME", 1, /DATABASE_URL/u],
38
+ [{ PROD_DB_NAME: "catalogue_test_2" }, "TEST_DB_NAME", 3, /PROD_DB_NAME \(catalogue_test_2\)/u],
39
+ [{ BROWSER_TEST_DB_NAME: "catalogue_test" }, "TEST_DB_NAME", 1, /the other test database/u],
40
+ [{ BROWSER_TEST_DB_NAME: "catalogue_test_1" }, "TEST_DB_NAME", 2, /the other test database/u],
41
+ [{ TEST_DB_NAME: "catalogue_browser_2" }, "BROWSER_TEST_DB_NAME", 1, /the other test database/u],
42
+ [{ NODE_ENV: "production" }, "TEST_DB_NAME", 1, /NODE_ENV=production/u]
43
+ ]) {
44
+ assert.throws(() => requireDisposableDatabase({ ...environment, ...overrides }, variable, count), message,
45
+ JSON.stringify(overrides));
46
+ }
47
+ // The database the Knex configuration connects to, whatever the environment says.
48
+ assert.throws(() => requireDisposableDatabase({ ...environment, DB_NAME: "" }, "TEST_DB_NAME", 2, { connected: "Catalogue_Test_1" }),
49
+ /PROD_DB_NAME \(catalogue_test_1\)/u);
50
+ });
51
+
52
+ test("the snapshot key changes when a migration changes, is added or is renamed, and only then", async (context) => {
53
+ const appRoot = await mkdtemp(path.join(os.tmpdir(), "jskit-test-databases-"));
54
+ context.after(() => rm(appRoot, { recursive: true, force: true }));
55
+ await mkdir(path.join(appRoot, "migrations", "constraints"), { recursive: true });
56
+ await writeFile(path.join(appRoot, "migrations", "20200101000000_books.cjs"), MIGRATION);
57
+ const knexConfig = createKnexMigrationConfig({ client: "pg", environment: connectionEnvironment, appRoot });
58
+
59
+ const first = await testDatabaseSnapshotKey(knexConfig);
60
+ assert.match(first.key, /^[0-9a-f]{32}$/u);
61
+ assert.deepEqual(first.migrations, ["20200101000000_books.cjs"]);
62
+ assert.deepEqual(await testDatabaseSnapshotKey(knexConfig), first);
63
+
64
+ const keys = new Set([first.key]);
65
+ await writeFile(path.join(appRoot, "migrations", "20200101000000_books.cjs"), MIGRATION.replace("increments()", "increments(\"book_id\")"));
66
+ keys.add((await testDatabaseSnapshotKey(knexConfig)).key);
67
+ await writeFile(path.join(appRoot, "migrations", "constraints", "20200102000000_books_index.cjs"), MIGRATION);
68
+ const added = await testDatabaseSnapshotKey(knexConfig);
69
+ keys.add(added.key);
70
+ assert.deepEqual(added.migrations, ["20200101000000_books.cjs", "20200102000000_books_index.cjs"]);
71
+ await rename(
72
+ path.join(appRoot, "migrations", "20200101000000_books.cjs"),
73
+ path.join(appRoot, "migrations", "20200101000001_books.cjs")
74
+ );
75
+ keys.add((await testDatabaseSnapshotKey(knexConfig)).key);
76
+ keys.add((await testDatabaseSnapshotKey({ ...knexConfig, client: "mysql2" })).key);
77
+ assert.equal(keys.size, 5);
78
+
79
+ // Installing other packages can change what their migrations' helpers build.
80
+ const before = await testDatabaseSnapshotKey(knexConfig, { appRoot });
81
+ await writeFile(path.join(appRoot, "package-lock.json"), "{\"lockfileVersion\":3}\n");
82
+ const locked = await testDatabaseSnapshotKey(knexConfig, { appRoot });
83
+ assert.notEqual(locked.key, before.key);
84
+ assert.deepEqual(await testDatabaseSnapshotKey(knexConfig, { appRoot }), locked);
85
+
86
+ // A schema build's own files join the key: a changed baseline takes a new snapshot.
87
+ await writeFile(path.join(appRoot, "baseline.sql"), "CREATE TABLE shelves (id INT);\n");
88
+ const built = await testDatabaseSnapshotKey(knexConfig, { appRoot, schemaFiles: ["baseline.sql"] });
89
+ assert.notEqual(built.key, locked.key);
90
+ assert.deepEqual(built.migrations, locked.migrations);
91
+ assert.deepEqual(await testDatabaseSnapshotKey(knexConfig, { appRoot, schemaFiles: ["baseline.sql"] }), built);
92
+ await writeFile(path.join(appRoot, "baseline.sql"), "CREATE TABLE shelves (id BIGINT);\n");
93
+ assert.notEqual((await testDatabaseSnapshotKey(knexConfig, { appRoot, schemaFiles: ["baseline.sql"] })).key, built.key);
94
+ });
95
+
96
+ test("with no usable test database, a run's test files run one at a time on the connected database", async () => {
97
+ const knexConfig = createKnexMigrationConfig({ client: "mysql2", environment: connectionEnvironment, appRoot: "/srv/catalogue" });
98
+ for (const [environment, reason] of [
99
+ [connectionEnvironment, /TEST_DB_NAME is not set/u],
100
+ [{ ...connectionEnvironment, TEST_DB_NAME: "CATALOGUE" }, /the connected database is TEST_DB_NAME \(CATALOGUE\)/u]
101
+ ]) {
102
+ const run = await testDatabaseRun({ knexConfig, environment, count: 4 });
103
+ assert.equal(run.concurrency, 1);
104
+ assert.deepEqual(run.env, {});
105
+ assert.equal(run.preload, "");
106
+ assert.match(run.summary, /^test files run one at a time on the connected database: /u);
107
+ assert.match(run.summary, reason);
108
+ }
109
+ });
110
+
111
+ test("a test database copy named like a protected database fails the run instead of running on it", async () => {
112
+ const knexConfig = createKnexMigrationConfig({ client: "mysql2", environment: connectionEnvironment, appRoot: "/srv/catalogue" });
113
+ const environment = { ...connectionEnvironment, TEST_DB_NAME: "catalogue_test", PROD_DB_NAME: "catalogue_test_1" };
114
+ await assert.rejects(testDatabaseRun({ knexConfig, environment, count: 4 }), /PROD_DB_NAME \(catalogue_test_1\)/u);
115
+ });
116
+
117
+ test("countQueries() records each statement a piece of work sends, so a read per row stands out from a batched read", async () => {
118
+ const knex = knexFactory({ client: "better-sqlite3", connection: { filename: ":memory:" }, useNullAsDefault: true });
119
+ try {
120
+ await knex.schema.createTable("books", (table) => { table.increments("id"); table.string("title"); });
121
+ await knex("books").insert(Array.from({ length: 30 }, (_, index) => ({ title: `Book ${index + 1}` })));
122
+ const ids = await knex("books").orderBy("id").pluck("id");
123
+ const titleEach = (some) => countQueries(knex, async () => {
124
+ const titles = [];
125
+ for (const id of some) titles.push((await knex("books").where({ id }).first("title")).title);
126
+ return titles;
127
+ });
128
+ const titlesAtOnce = (some) => countQueries(knex, () => knex("books").whereIn("id", some).orderBy("id").pluck("title"));
129
+
130
+ const few = await titleEach(ids.slice(0, 3));
131
+ assert.deepEqual(few, {
132
+ result: ["Book 1", "Book 2", "Book 3"],
133
+ count: 3,
134
+ queries: Array(3).fill("select `title` from `books` where `id` = ? limit ?")
135
+ });
136
+ assert.equal((await titleEach(ids)).count, 30);
137
+ assert.deepEqual([(await titlesAtOnce(ids.slice(0, 3))).count, (await titlesAtOnce(ids)).count], [1, 1]);
138
+
139
+ // A transaction's statements reach the Knex instance it was opened from; counting on the transaction works too.
140
+ const inTransaction = await countQueries(knex, () => knex.transaction(async (trx) => {
141
+ const own = await countQueries(trx, () => trx("books").where({ id: ids[0] }).first("title"));
142
+ assert.equal(own.count, 1);
143
+ return own.result;
144
+ }));
145
+ assert.deepEqual(inTransaction.result, { title: "Book 1" });
146
+ assert.deepEqual(inTransaction.queries, ["BEGIN;", "select `title` from `books` where `id` = ? limit ?", "COMMIT;"]);
147
+
148
+ // Counting ends with the work, failed work included.
149
+ const listeners = knex.listenerCount("query");
150
+ await assert.rejects(countQueries(knex, async () => {
151
+ await knex("books").count({ books: "*" });
152
+ throw new Error("stopped");
153
+ }), /stopped/u);
154
+ assert.equal(knex.listenerCount("query"), listeners);
155
+ } finally {
156
+ await knex.destroy();
157
+ }
158
+ });