@procedures/sqlite 0.2.3 → 0.3.0

package/src/procedures/db/migrate.ts

@@ -0,0 +1,121 @@
+/**
+ * db.migrate procedure (roadmap R44)
+ *
+ * The procedure applies the numbered migrations that the database does not have, in the order
+ * of their numbers. The migrations come from a folder (files named `<number>_<name>.sql`) or
+ * from the input. A table of the database (default `schema_migrations`) records each applied
+ * migration with the sha256 of its SQL.
+ *
+ * - The pending migrations run in one transaction: when one fails, none is applied.
+ * - A second call applies nothing, and does not write the file.
+ * - The SQL of an applied migration must not change: a different sha256 is an error, and the
+ *   call applies nothing. (The sha256 reads line ends as LF, so a checkout with CRLF is the same.)
+ * - A migration must not hold BEGIN, COMMIT or ROLLBACK: the procedure runs the transaction.
+ */
+
+import { createHash } from "node:crypto";
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { query, withConnection } from "@mark1russell7/docker-sqlite";
+import { MIGRATION_TABLE_PATTERN, type DbMigrateInput, type DbMigrateOutput, type DbMigration } from "../../types.js";
+import { databasePath, runStatements } from "./shared.js";
+
+/** The name of a migration file: a number, then "_" or "-", then a name, then ".sql". */
+const MIGRATION_FILE = /^(\d+)[_-](.+)\.sql$/i;
+
+/** The default table of the applied migrations. */
+export const DEFAULT_MIGRATION_TABLE = "schema_migrations";
+
+interface Migration {
+  version: number;
+  name: string;
+  sql: string;
+  checksum: string;
+}
+
+function checksum(sql: string): string {
+  return createHash("sha256").update(sql.replace(/\r\n/g, "\n")).digest("hex");
+}
+
+/** The migrations of a folder: each file `<number>_<name>.sql`. Other files are not migrations. */
+export async function migrationsOfFolder(folder: string): Promise<DbMigration[]> {
+  const entries = await readdir(folder, { withFileTypes: true });
+  const migrations: DbMigration[] = [];
+  for (const entry of entries) {
+    if (!entry.isFile() || !entry.name.toLowerCase().endsWith(".sql")) continue;
+    const match = MIGRATION_FILE.exec(entry.name);
+    if (!match) throw new Error(`db.migrate: the file ${entry.name} has no number: name it <number>_<name>.sql`);
+    migrations.push({ version: Number(match[1]), name: match[2]!, sql: await readFile(join(folder, entry.name), "utf8") });
+  }
+  return migrations;
+}
+
+/** The migrations in the order of their numbers. Two migrations with one number are an error. */
+function ordered(migrations: DbMigration[]): Migration[] {
+  const byVersion = new Map<number, Migration>();
+  for (const migration of migrations) {
+    if (!Number.isSafeInteger(migration.version) || migration.version < 1) {
+      throw new Error(`db.migrate: the version ${migration.version} is not a whole number of 1 or more`);
+    }
+    if (byVersion.has(migration.version)) throw new Error(`db.migrate: two migrations have the version ${migration.version}`);
+    const name = migration.name ?? `migration ${migration.version}`;
+    byVersion.set(migration.version, { version: migration.version, name, sql: migration.sql, checksum: checksum(migration.sql) });
+  }
+  return [...byVersion.values()].sort((a, b) => a.version - b.version);
+}
+
+/**
+ * Apply the numbered migrations that a database does not have
+ */
+export async function dbMigrate(input: DbMigrateInput): Promise<DbMigrateOutput> {
+  if ((input.folder === undefined) === (input.migrations === undefined)) {
+    throw new Error("db.migrate: give exactly one of folder or migrations");
+  }
+  const table = input.table ?? DEFAULT_MIGRATION_TABLE;
+  if (!MIGRATION_TABLE_PATTERN.test(table)) throw new Error(`db.migrate: the table name ${JSON.stringify(table)} is not valid`);
+  const migrations = ordered(input.folder !== undefined ? await migrationsOfFolder(input.folder) : input.migrations!);
+
+  return withConnection({ dbPath: databasePath(input.dbPath) }, (db) => {
+    db.run("BEGIN");
+    try {
+      db.run(
+        `CREATE TABLE IF NOT EXISTS "${table}" (version INTEGER PRIMARY KEY, name TEXT NOT NULL, checksum TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT (datetime('now')))`,
+      );
+      const applied = new Map(
+        query<{ version: number; checksum: string }>(db, `SELECT version, checksum FROM "${table}"`).rows.map((row) => [row.version, row.checksum]),
+      );
+      for (const migration of migrations) {
+        const known = applied.get(migration.version);
+        if (known !== undefined && known !== migration.checksum) {
+          throw new Error(`db.migrate: the migration ${migration.version} (${migration.name}) changed after it was applied: its sha256 is different`);
+        }
+      }
+      const pending = migrations.filter((migration) => !applied.has(migration.version));
+      for (const migration of pending) {
+        const label = `the migration ${migration.version} (${migration.name})`;
+        try {
+          runStatements(db, migration.sql, [], label);
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          throw new Error(message.startsWith(label) ? `db.migrate: ${message}` : `db.migrate: ${label} failed: ${message}`);
+        }
+        db.run(`INSERT INTO "${table}" (version, name, checksum) VALUES (?, ?, ?)`, [migration.version, migration.name, migration.checksum]);
+      }
+      db.run("COMMIT");
+      const versions = [...applied.keys(), ...pending.map((migration) => migration.version)];
+      return {
+        applied: pending.map((migration) => ({ version: migration.version, name: migration.name })),
+        skipped: migrations.length - pending.length,
+        version: versions.length > 0 ? Math.max(...versions) : 0,
+      };
+    } catch (error) {
+      try {
+        db.run("ROLLBACK");
+      } catch {
+        // SQLite ended the transaction already
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`${message}. No migration was applied.`);
+    }
+  });
+}

package/src/procedures/db/schema.ts

@@ -0,0 +1,71 @@
+/**
+ * db.schema procedure (roadmap R44)
+ *
+ * The columns, the indexes and the foreign keys of a table (or the columns of a view). The
+ * procedure reads them with the table-valued PRAGMA functions, so the name of the table is a
+ * bound value, not a part of the SQL text. The connection is read only.
+ */
+
+import { query, withConnection } from "@mark1russell7/docker-sqlite";
+import type { DbColumnInfo, DbForeignKeyInfo, DbIndexInfo, DbSchemaInput, DbSchemaOutput } from "../../types.js";
+import { databasePath, requireDatabaseFile } from "./shared.js";
+
+interface ColumnRow {
+  name: string;
+  type: string;
+  notnull: number;
+  dflt_value: string | null;
+  pk: number;
+}
+
+interface IndexRow {
+  name: string;
+  unique: number;
+  origin: string;
+  partial: number;
+}
+
+interface ForeignKeyRow {
+  from: string;
+  table: string;
+  to: string | null;
+  on_update: string;
+  on_delete: string;
+}
+
+/**
+ * The columns and indexes of a table
+ */
+export async function dbSchema(input: DbSchemaInput): Promise<DbSchemaOutput> {
+  const path = databasePath(input.dbPath);
+  requireDatabaseFile("db.schema", path);
+  return withConnection({ dbPath: path }, (db) => {
+    db.run("PRAGMA query_only = 1");
+    const found = query<{ type: "table" | "view"; sql: string | null }>(
+      db,
+      "SELECT type, sql FROM sqlite_master WHERE name = ? AND type IN ('table', 'view')",
+      [input.table],
+    ).rows[0];
+    if (!found) throw new Error(`db.schema: no table or view named ${JSON.stringify(input.table)}`);
+
+    const columns: DbColumnInfo[] = query<ColumnRow>(db, "SELECT name, type, \"notnull\", dflt_value, pk FROM pragma_table_info(?) ORDER BY cid", [input.table]).rows.map(
+      (row) => ({ name: row.name, type: row.type, notNull: row.notnull === 1, defaultValue: row.dflt_value, primaryKey: row.pk }),
+    );
+    const indexes: DbIndexInfo[] = query<IndexRow>(db, "SELECT name, \"unique\", origin, partial FROM pragma_index_list(?) ORDER BY name", [input.table]).rows.map(
+      (row) => ({
+        name: row.name,
+        unique: row.unique === 1,
+        origin: row.origin,
+        partial: row.partial === 1,
+        columns: query<{ name: string | null }>(db, "SELECT name FROM pragma_index_info(?) ORDER BY seqno", [row.name]).rows.map((column) => column.name ?? "<expression>"),
+      }),
+    );
+    const foreignKeys: DbForeignKeyInfo[] = query<ForeignKeyRow>(
+      db,
+      "SELECT \"from\", \"table\", \"to\", on_update, on_delete FROM pragma_foreign_key_list(?) ORDER BY id, seq",
+      [input.table],
+    ).rows.map((row) => ({ from: row.from, table: row.table, to: row.to, onUpdate: row.on_update, onDelete: row.on_delete }));
+
+    return { table: input.table, type: found.type, sql: found.sql, columns, indexes, foreignKeys };
+  });
+}

package/src/procedures/db/shared.ts

@@ -0,0 +1,109 @@
+/**
+ * The helpers of the db.* procedures of roadmap R44.
+ */
+
+import { existsSync } from "node:fs";
+import type { Database } from "@mark1russell7/docker-sqlite";
+import { DEFAULT_DB_PATH } from "../../types.js";
+
+/** The database file of an input: `dbPath`, or the CLI log database. */
+export function databasePath(dbPath: string | undefined): string {
+  return dbPath ?? DEFAULT_DB_PATH;
+}
+
+/**
+ * This function throws ENOENT for a database file that does not exist. A procedure that only
+ * reads uses it: sql.js opens a missing file as an empty database, so a wrong path gave no tables.
+ */
+export function requireDatabaseFile(procedure: string, path: string): void {
+  if (path === ":memory:" || existsSync(path)) return;
+  const error = new Error(`${procedure}: the database file does not exist: ${path}`) as NodeJS.ErrnoException;
+  error.code = "ENOENT";
+  throw error;
+}
+
+/** The statements that control a transaction, which the procedures run themselves. */
+const TRANSACTION_CONTROL = /^(BEGIN|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE)\b/i;
+
+/** The SQL of a statement without its leading spaces and comments. */
+function withoutLeadingComments(sql: string): string {
+  let rest = sql;
+  for (;;) {
+    const trimmed = rest.replace(/^\s+/, "");
+    if (trimmed.startsWith("--")) {
+      const end = trimmed.indexOf("\n");
+      rest = end === -1 ? "" : trimmed.slice(end + 1);
+    } else if (trimmed.startsWith("/*")) {
+      const end = trimmed.indexOf("*/");
+      rest = end === -1 ? "" : trimmed.slice(end + 2);
+    } else {
+      return trimmed;
+    }
+  }
+}
+
+/** True for a statement that starts, ends or names a transaction (BEGIN, COMMIT, SAVEPOINT, ...). */
+export function controlsTransaction(sql: string): boolean {
+  return TRANSACTION_CONTROL.test(withoutLeadingComments(sql));
+}
+
+/** The result of one statement: the rows that it changed, and its rows when it gives rows. */
+export interface StatementRun {
+  changes: number;
+  rows?: Record<string, unknown>[];
+}
+
+function totalChanges(db: Database): number {
+  return Number(db.exec("SELECT total_changes()")[0]?.values[0]?.[0] ?? 0);
+}
+
+/**
+ * This function runs the statements of one SQL text in order, and gives one result for the text.
+ * The params bind to the first statement, so a text with params must hold one statement. A
+ * statement that controls a transaction is refused: the caller owns the transaction.
+ */
+export function runStatements(db: Database, sql: string, params: unknown[], label: string): StatementRun {
+  const before = totalChanges(db);
+  const result: StatementRun = { changes: 0 };
+  let count = 0;
+  const iterator = db.iterateStatements(sql);
+  try {
+    for (let step = iterator.next(); !step.done; step = iterator.next()) {
+      const statement = step.value;
+      try {
+        count++;
+        if (count > 1 && params.length > 0) {
+          throw new Error(`${label}: the params bind to one statement, and the SQL has several`);
+        }
+        if (controlsTransaction(statement.getSQL())) {
+          throw new Error(`${label}: the procedure runs the transaction: BEGIN, COMMIT, ROLLBACK, SAVEPOINT and RELEASE are refused`);
+        }
+        if (count === 1 && params.length > 0) statement.bind(params as never);
+        if (statement.getColumnNames().length > 0) {
+          const rows: Record<string, unknown>[] = [];
+          while (statement.step()) rows.push(statement.getAsObject() as Record<string, unknown>);
+          result.rows = rows;
+        } else {
+          while (statement.step()) {
+            // A statement without columns gives no rows: one step runs it
+          }
+        }
+      } finally {
+        statement.free();
+      }
+    }
+  } catch (error) {
+    // Read the iterator to its end: that frees its copy of the SQL (it only prepares the rest)
+    try {
+      while (!iterator.next().done) {
+        // Prepare only
+      }
+    } catch {
+      // A later statement did not prepare: the iterator ended
+    }
+    throw error;
+  }
+  if (count === 0) throw new Error(`${label}: the SQL has no statement`);
+  result.changes = totalChanges(db) - before;
+  return result;
+}

package/src/procedures/db/tables.ts

@@ -0,0 +1,27 @@
+/**
+ * db.tables procedure (roadmap R44)
+ *
+ * The tables and the views of a database, without the internal `sqlite_` tables. The connection
+ * is read only (PRAGMA query_only), and a missing file is an error: it does not make one.
+ */
+
+import { query, withConnection } from "@mark1russell7/docker-sqlite";
+import type { DbTableInfo, DbTablesInput, DbTablesOutput } from "../../types.js";
+import { databasePath, requireDatabaseFile } from "./shared.js";
+
+/**
+ * The tables and views of a database
+ */
+export async function dbTables(input: DbTablesInput): Promise<DbTablesOutput> {
+  const path = databasePath(input.dbPath);
+  requireDatabaseFile("db.tables", path);
+  const types = input.views === false ? "('table')" : "('table', 'view')";
+  return withConnection({ dbPath: path }, (db) => {
+    db.run("PRAGMA query_only = 1");
+    const result = query<DbTableInfo>(
+      db,
+      `SELECT name, type, sql FROM sqlite_master WHERE type IN ${types} AND name NOT LIKE 'sqlite\\_%' ESCAPE '\\' ORDER BY name`,
+    );
+    return { tables: result.rows.map((row) => ({ name: row.name, type: row.type, sql: row.sql ?? null })) };
+  });
+}

package/src/procedures/db/transaction.ts

@@ -0,0 +1,43 @@
+/**
+ * db.transaction procedure (roadmap R44)
+ *
+ * The statements of the input run in order in one transaction. When one fails, the procedure
+ * rolls back the transaction and does not save the file: the database stays as it was. A
+ * statement that gives rows (SELECT, or RETURNING) gives them in its result, so a later
+ * statement can follow an earlier one in the same transaction.
+ */
+
+import { withConnection } from "@mark1russell7/docker-sqlite";
+import type { DbStatementResult, DbTransactionInput, DbTransactionOutput } from "../../types.js";
+import { databasePath, runStatements } from "./shared.js";
+
+/**
+ * Run statements in one transaction
+ */
+export async function dbTransaction(input: DbTransactionInput): Promise<DbTransactionOutput> {
+  return withConnection({ dbPath: databasePath(input.dbPath) }, (db) => {
+    db.run("BEGIN");
+    const results: DbStatementResult[] = [];
+    try {
+      input.statements.forEach((statement, index) => {
+        const label = `db.transaction: statements[${index}]`;
+        try {
+          results.push(runStatements(db, statement.sql, statement.params ?? [], label));
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error);
+          throw new Error(message.startsWith(label) ? message : `${label}: ${message}`);
+        }
+      });
+      db.run("COMMIT");
+    } catch (error) {
+      try {
+        db.run("ROLLBACK");
+      } catch {
+        // SQLite ended the transaction already (an ON CONFLICT ROLLBACK)
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`${message}. The transaction was rolled back.`);
+    }
+    return { results, changes: results.reduce((sum, result) => sum + result.changes, 0) };
+  });
+}