@mark1russell7/docker-sqlite 0.0.0-stage → 0.1.1

package/src/connect.ts

@@ -0,0 +1,245 @@
+/**
+ * Connection utilities for SQLite using sql.js
+ */
+
+import initSqlJs, { type Database } from "sql.js";
+import { randomUUID } from "node:crypto";
+import { readFile, mkdir, open, rename, unlink } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import type { ConnectionConfig, ConnectionCallback } from "./types.js";
+
+let sqlPromise: Promise<Awaited<ReturnType<typeof initSqlJs>>> | null = null;
+
+/**
+ * Initialize sql.js (cached)
+ */
+async function getSql(): Promise<Awaited<ReturnType<typeof initSqlJs>>> {
+  if (!sqlPromise) {
+    sqlPromise = initSqlJs();
+  }
+  return sqlPromise;
+}
+
+/** A database and the bytes of its file when it was loaded (null: there was no file). */
+interface LoadedDatabase {
+  db: Database;
+  original: Uint8Array | null;
+}
+
+/**
+ * Load a database from a file path or create a new one
+ */
+async function loadDatabase(dbPath: string): Promise<LoadedDatabase> {
+  const SQL = await getSql();
+
+  if (dbPath === ":memory:") {
+    return { db: new SQL.Database(), original: null };
+  }
+
+  try {
+    const buffer = await readFile(dbPath);
+    // sql.js keeps the array that it gets and writes into it: give it a copy, so `original`
+    // keeps the bytes of the file
+    return { db: new SQL.Database(new Uint8Array(buffer)), original: buffer };
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
+      // File doesn't exist, create new database
+      return { db: new SQL.Database(), original: null };
+    }
+    throw error;
+  }
+}
+
+/**
+ * Save a database to a file path, when its content changed.
+ *
+ * A database that did not change is not written. Before, each call wrote the file, also a
+ * read: a read failed on a read-only file (EPERM), a read wrote an old copy over the rows of
+ * another process, and a query of a missing file made an empty file. The export of a
+ * database that did not change has the bytes of its file, and an empty new database exports
+ * no bytes.
+ *
+ * Writes a temporary file, flushes it to the disk and renames it over the database, so a crash
+ * or a power loss during the write cannot leave a truncated database (the rename replaces the
+ * file in one step).
+ */
+async function saveDatabase(db: Database, dbPath: string, original: Uint8Array | null): Promise<Uint8Array | null> {
+  if (dbPath === ":memory:") {
+    return null;
+  }
+
+  const data = db.export();
+  const unchanged = original
+    ? Buffer.compare(Buffer.from(data), Buffer.from(original)) === 0
+    : data.length === 0;
+  if (unchanged) {
+    return original;
+  }
+  const buffer = Buffer.from(data);
+
+  // Ensure directory exists
+  await mkdir(dirname(dbPath), { recursive: true });
+
+  // A random part, so two saves at once never share a temporary file. Before, the name had only
+  // the process id and the time in milliseconds.
+  const tempPath = `${dbPath}.${process.pid}.${randomUUID()}.tmp`;
+  try {
+    await writeDurably(tempPath, buffer);
+  } catch (error) {
+    await unlink(tempPath).catch(() => {});
+    throw error;
+  }
+  try {
+    await rename(tempPath, dbPath);
+  } catch (error) {
+    await unlink(tempPath).catch(() => {});
+    // The file is held open on Windows: fall back to a direct write. Before, each error of the
+    // rename started the direct write, which is not atomic.
+    if (!FILE_IN_USE.has((error as NodeJS.ErrnoException).code ?? "")) {
+      throw error;
+    }
+    await writeDurably(dbPath, buffer);
+  }
+  await syncDirectory(dirname(dbPath));
+  return data;
+}
+
+/** The error codes of a rename over a file that another process holds open (Windows). */
+const FILE_IN_USE = new Set(["EPERM", "EACCES", "EBUSY"]);
+
+/**
+ * Write a file and flush it to the disk before the rename. Without the flush, a power loss soon
+ * after the rename can leave an empty or partial database on some file systems.
+ */
+async function writeDurably(path: string, data: Uint8Array): Promise<void> {
+  const file = await open(path, "w");
+  try {
+    await file.writeFile(data);
+    await file.sync();
+  } finally {
+    await file.close();
+  }
+}
+
+/**
+ * Flush a directory, so that a rename in it stays after a power loss. Windows cannot open a
+ * directory for this, and some file systems refuse it: the flush is only an extra safety.
+ */
+async function syncDirectory(path: string): Promise<void> {
+  if (process.platform === "win32") {
+    return;
+  }
+  try {
+    const directory = await open(path, "r");
+    try {
+      await directory.sync();
+    } finally {
+      await directory.close();
+    }
+  } catch {
+    // The rename is done
+  }
+}
+
+/**
+ * The calls that wait for each database file. Each call reads the whole file, changes it in
+ * memory and writes it back, so two calls at once lost the changes of one of them
+ * (50 concurrent inserts kept 1 row). Calls for the same file now run one at a time in this
+ * process. (Other processes that write the same file are not coordinated.)
+ */
+const fileQueues = new Map<string, Promise<unknown>>();
+
+/**
+ * The queue key of a file. Windows file names do not depend on case, so the key is in lower
+ * case there. Before, two spellings of one file had two queues, and their calls ran at once.
+ */
+export function fileQueueKey(dbPath: string, platform: NodeJS.Platform = process.platform): string {
+  const path = resolve(dbPath);
+  return platform === "win32" ? path.toLowerCase() : path;
+}
+
+async function inFileQueue<T>(dbPath: string, task: () => Promise<T>): Promise<T> {
+  if (dbPath === ":memory:") {
+    return task();
+  }
+  const key = fileQueueKey(dbPath);
+  const previous = fileQueues.get(key) ?? Promise.resolve();
+  const run = previous.then(task, task);
+  const tail = run.then(
+    () => undefined,
+    () => undefined
+  );
+  fileQueues.set(key, tail);
+  try {
+    return await run;
+  } finally {
+    if (fileQueues.get(key) === tail) {
+      fileQueues.delete(key);
+    }
+  }
+}
+
+/**
+ * Execute a callback with a SQLite connection, ensuring proper cleanup and persistence.
+ *
+ * @example
+ * ```typescript
+ * const result = await withConnection(
+ *   { dbPath: './data/app.db' },
+ *   async (db) => {
+ *     const stmt = db.prepare('SELECT * FROM users WHERE id = ?');
+ *     stmt.bind([1]);
+ *     const rows = [];
+ *     while (stmt.step()) {
+ *       rows.push(stmt.getAsObject());
+ *     }
+ *     stmt.free();
+ *     return rows;
+ *   }
+ * );
+ * ```
+ */
+export async function withConnection<T>(
+  config: ConnectionConfig,
+  fn: ConnectionCallback<T>
+): Promise<T> {
+  return inFileQueue(config.dbPath, async () => {
+    const { db, original } = await loadDatabase(config.dbPath);
+
+    try {
+      const result = await fn(db);
+      // Save changes back to file (only when there are changes)
+      await saveDatabase(db, config.dbPath, original);
+      return result;
+    } finally {
+      db.close();
+    }
+  });
+}
+
+/**
+ * Create a SQLite database that can be manually managed.
+ * Caller is responsible for calling db.close() and saving if needed.
+ *
+ * @example
+ * ```typescript
+ * const { db, save } = await createConnection({ dbPath: './data/app.db' });
+ * try {
+ *   db.run('INSERT INTO users (name) VALUES (?)', ['Alice']);
+ *   await save();
+ * } finally {
+ *   db.close();
+ * }
+ * ```
+ */
+export async function createConnection(
+  config: ConnectionConfig
+): Promise<{ db: Database; save: () => Promise<void> }> {
+  const { db, original } = await loadDatabase(config.dbPath);
+  // The bytes of the file after the last save, to compare the next save with
+  let saved = original;
+  const save = async (): Promise<void> => {
+    saved = await saveDatabase(db, config.dbPath, saved);
+  };
+  return { db, save };
+}

package/src/index.ts

@@ -0,0 +1,61 @@
+/**
+ * docker-sqlite - Reusable SQLite Docker setup with TypeScript utilities
+ *
+ * @example
+ * ```typescript
+ * import { withConnection, query, execute, runMigrations } from 'docker-sqlite';
+ *
+ * // Define migrations
+ * const migrations = [
+ *   {
+ *     version: 1,
+ *     description: 'Create users table',
+ *     up: 'CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, email TEXT)',
+ *     down: 'DROP TABLE users'
+ *   }
+ * ];
+ *
+ * // Run with auto-save
+ * await withConnection({ dbPath: './data/app.db' }, (db) => {
+ *   runMigrations(db, migrations);
+ *
+ *   execute(db, 'INSERT INTO users (name, email) VALUES (?, ?)', ['Alice', 'alice@example.com']);
+ *
+ *   const result = query(db, 'SELECT * FROM users');
+ *   console.log(result.rows);
+ * });
+ * ```
+ */
+
+// Types
+export type {
+  ConnectionConfig,
+  ConnectionCallback,
+  QueryResult,
+  ExecuteResult,
+  Migration,
+  MigrationStatus,
+} from "./types.js";
+
+// Connection utilities
+export { withConnection, createConnection } from "./connect.js";
+
+// Query utilities
+export {
+  query,
+  execute,
+  execMultiple,
+  lastInsertRowId,
+  tableExists,
+} from "./query.js";
+
+// Migration utilities
+export {
+  getMigrationStatus,
+  runMigrations,
+  rollbackMigration,
+  createMigration,
+} from "./migrate.js";
+
+// Re-export sql.js Database type for convenience
+export type { Database } from "sql.js";

package/src/migrate.ts

@@ -0,0 +1,293 @@
+/**
+ * Migration utilities for SQLite
+ */
+
+import type { Database } from "sql.js";
+import type { Migration, MigrationStatus } from "./types.js";
+import { query, execute, execMultiple } from "./query.js";
+
+/**
+ * Schema for tracking migrations
+ */
+const MIGRATIONS_TABLE_SQL = `
+  CREATE TABLE IF NOT EXISTS _migrations (
+    version INTEGER PRIMARY KEY,
+    description TEXT NOT NULL,
+    applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+  )
+`;
+
+/**
+ * Ensure the migrations table exists
+ */
+function ensureMigrationsTable(db: Database): void {
+  execMultiple(db, MIGRATIONS_TABLE_SQL);
+}
+
+/** The name of the savepoint that holds one migration and its record. */
+const SAVEPOINT = "docker_sqlite_migration";
+
+/**
+ * The first words of the statements that control transactions, cannot run in a transaction
+ * (VACUUM, ATTACH) or can do nothing there (PRAGMA foreign_keys).
+ */
+const OUTSIDE_A_TRANSACTION = new Set([
+  "BEGIN",
+  "COMMIT",
+  "END",
+  "ROLLBACK",
+  "SAVEPOINT",
+  "RELEASE",
+  "VACUUM",
+  "ATTACH",
+  "DETACH",
+  "PRAGMA",
+]);
+
+/**
+ * The tokens of SQL text: the words in upper case, ";" for each semicolon, and "?" for each
+ * other token. Comments are skipped. A string or a quoted name is one "?" token, so a word in it
+ * is not a keyword.
+ */
+function* tokens(sql: string): Generator<string> {
+  const word = /[\p{L}_][\p{L}\p{N}_$]*/uy;
+  let i = 0;
+  while (i < sql.length) {
+    const c = sql[i]!;
+    if (/\s/.test(c)) {
+      i++;
+    } else if (c === "-" && sql[i + 1] === "-") {
+      const end = sql.indexOf("\n", i);
+      i = end < 0 ? sql.length : end + 1;
+    } else if (c === "/" && sql[i + 1] === "*") {
+      const end = sql.indexOf("*/", i + 2);
+      i = end < 0 ? sql.length : end + 2;
+    } else if (c === "'" || c === '"' || c === "`" || c === "[") {
+      // A quote in the text is two quotes. A name in brackets cannot hold "]".
+      const close = c === "[" ? "]" : c;
+      let end = sql.indexOf(close, i + 1);
+      while (end >= 0 && close !== "]" && sql[end + 1] === close) {
+        end = sql.indexOf(close, end + 2);
+      }
+      i = end < 0 ? sql.length : end + 1;
+      yield "?";
+    } else if (c === ";") {
+      i++;
+      yield ";";
+    } else {
+      word.lastIndex = i;
+      const match = word.exec(sql);
+      if (match) {
+        i += match[0].length;
+        yield match[0].toUpperCase();
+      } else {
+        i++;
+        yield "?";
+      }
+    }
+  }
+}
+
+/**
+ * The first word of each statement of `sql`, in upper case. The statements in the body of a
+ * trigger are part of the trigger: like sqlite3_complete(), the trigger ends at "; END ;".
+ */
+function statementKeywords(sql: string): string[] {
+  const keywords: string[] = [];
+  let words: string[] = [];
+  let trigger = false;
+  let afterSemicolon = false;
+  let afterEnd = false;
+  for (const token of tokens(sql)) {
+    if (token === ";") {
+      if (!trigger || afterEnd) {
+        words = [];
+        trigger = false;
+        afterEnd = false;
+      } else {
+        afterSemicolon = true;
+      }
+      continue;
+    }
+    if (words.length === 0) {
+      keywords.push(token);
+    }
+    if (words.length < 3) {
+      words.push(token);
+      const [first, second, third] = words;
+      trigger ||=
+        first === "CREATE" &&
+        (second === "TRIGGER" || ((second === "TEMP" || second === "TEMPORARY") && third === "TRIGGER"));
+    }
+    afterEnd = trigger && afterSemicolon && token === "END";
+    afterSemicolon = false;
+  }
+  return keywords;
+}
+
+/**
+ * Run `work` in a savepoint: its changes stay only when it succeeds. Before, a migration that
+ * failed after its first statement kept the changes of those statements, and was not recorded:
+ * the next run ran it again and failed ("table already exists").
+ *
+ * A savepoint also works in a transaction of the caller, and keeps that transaction open. SQL
+ * that controls transactions itself (BEGIN, COMMIT, VACUUM, PRAGMA and the others in
+ * OUTSIDE_A_TRANSACTION) runs without a savepoint, as before: in a transaction it fails or does
+ * nothing.
+ */
+function atomically(db: Database, sql: string, work: () => void): void {
+  if (statementKeywords(sql).some((keyword) => OUTSIDE_A_TRANSACTION.has(keyword))) {
+    work();
+    return;
+  }
+  db.exec(`SAVEPOINT ${SAVEPOINT}`);
+  try {
+    work();
+    db.exec(`RELEASE ${SAVEPOINT}`);
+  } catch (error) {
+    try {
+      db.exec(`ROLLBACK TO ${SAVEPOINT}`);
+      db.exec(`RELEASE ${SAVEPOINT}`);
+    } catch {
+      // Some errors end the transaction (ON CONFLICT ROLLBACK): then the savepoint is gone
+    }
+    throw error;
+  }
+}
+
+/**
+ * Get the current migration status
+ */
+export function getMigrationStatus(
+  db: Database,
+  migrations: Migration[]
+): MigrationStatus {
+  ensureMigrationsTable(db);
+
+  const applied = query<{ version: number }>(
+    db,
+    "SELECT version FROM _migrations ORDER BY version"
+  );
+
+  const appliedVersions = new Set(applied.rows.map((r) => r.version));
+  const appliedMigrations = [...appliedVersions].sort((a, b) => a - b);
+  const currentVersion =
+    appliedMigrations.length > 0
+      ? appliedMigrations[appliedMigrations.length - 1]!
+      : 0;
+
+  const pendingMigrations = migrations
+    .filter((m) => !appliedVersions.has(m.version))
+    .sort((a, b) => a.version - b.version);
+
+  return {
+    currentVersion,
+    appliedMigrations,
+    pendingMigrations,
+  };
+}
+
+/**
+ * Run all pending migrations
+ *
+ * @example
+ * ```typescript
+ * const migrations: Migration[] = [
+ *   {
+ *     version: 1,
+ *     description: 'Create users table',
+ *     up: 'CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)',
+ *     down: 'DROP TABLE users'
+ *   },
+ *   {
+ *     version: 2,
+ *     description: 'Add email column',
+ *     up: 'ALTER TABLE users ADD COLUMN email TEXT',
+ *   }
+ * ];
+ *
+ * const applied = runMigrations(db, migrations);
+ * console.log(`Applied ${applied.length} migrations`);
+ * ```
+ */
+export function runMigrations(db: Database, migrations: Migration[]): Migration[] {
+  const status = getMigrationStatus(db, migrations);
+  const applied: Migration[] = [];
+
+  for (const migration of status.pendingMigrations) {
+    try {
+      // Run the migration and record it as applied, in one savepoint
+      atomically(db, migration.up, () => {
+        execMultiple(db, migration.up);
+        execute(
+          db,
+          "INSERT INTO _migrations (version, description) VALUES (?, ?)",
+          [migration.version, migration.description]
+        );
+      });
+
+      applied.push(migration);
+    } catch (error) {
+      throw new Error(
+        `Failed to apply migration ${migration.version} (${migration.description}): ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+  }
+
+  return applied;
+}
+
+/**
+ * Rollback the last applied migration
+ */
+export function rollbackMigration(
+  db: Database,
+  migrations: Migration[]
+): Migration | null {
+  const status = getMigrationStatus(db, migrations);
+
+  if (status.currentVersion === 0) {
+    return null;
+  }
+
+  const migration = migrations.find((m) => m.version === status.currentVersion);
+
+  if (!migration) {
+    throw new Error(`Migration ${status.currentVersion} not found in migrations list`);
+  }
+
+  if (!migration.down) {
+    throw new Error(
+      `Migration ${migration.version} (${migration.description}) has no down migration`
+    );
+  }
+
+  const down = migration.down;
+  try {
+    atomically(db, down, () => {
+      execMultiple(db, down);
+      execute(db, "DELETE FROM _migrations WHERE version = ?", [migration.version]);
+    });
+    return migration;
+  } catch (error) {
+    throw new Error(
+      `Failed to rollback migration ${migration.version}: ${error instanceof Error ? error.message : String(error)}`
+    );
+  }
+}
+
+/**
+ * Create a migration from SQL strings
+ */
+export function createMigration(
+  version: number,
+  description: string,
+  up: string,
+  down?: string
+): Migration {
+  const migration: Migration = { version, description, up };
+  if (down !== undefined) {
+    migration.down = down;
+  }
+  return migration;
+}