@apisurf/ngn-persistence 0.2.7 → 0.2.8

package/dist/client.d.ts

@@ -30,6 +30,15 @@ declare function applyConnectionPragmas(client: Client, { wal, busyTimeoutMs }?:
  * there is one place that decides how a connection behaves under contention.
  */
 declare function openDbClient(url: string, options?: ConnectionPragmaOptions): Client;
+/**
+ * Bring a file up to ngn's current schema. Returns the ids it applied.
+ *
+ * Everything pending goes in one write batch, so a file is never left halfway
+ * through a migration. If another process migrated the file between our read
+ * and our write, the batch fails on a table that already exists; reading again
+ * then finds nothing pending, and that is not an error.
+ */
+declare function migrate(dbClient: Client): Promise<string[]>;
 declare function initDbFileIfNotExists(dbPath: DbPath): Promise<DbPath>;
 /**
  * Connect to DB from a file path or throw an error if the file does not exist.
@@ -40,12 +49,12 @@ declare function strictInitDbClientFromFilePath(sqliteUrl: `file:${string}`): Pr
 /**
  * Initialize a database client from a SQLite URL.
  * If the URL is ':memory:', it creates an in-memory database.
- * If the URL is a file path, it checks if the file exists and runs migrations if it does not to
- * ensure the DB file is there and ready to use.
+ * If the URL is a file path, the file is created when missing. Either way the
+ * database is migrated to ngn's current schema before it is returned.
  * @param sqliteUrl
  * @returns
  */
 declare function safeInitDbClient(sqliteUrl: DbPath): Promise<Client>;
 declare function isDbPath(dbPath: string): dbPath is DbPath;
 
-export { type ConnectionPragmaOptions, type DbPath, applyConnectionPragmas, initDbFileIfNotExists, isDbPath, openDbClient, safeInitDbClient, strictInitDbClientFromFilePath };
+export { type ConnectionPragmaOptions, type DbPath, applyConnectionPragmas, initDbFileIfNotExists, isDbPath, migrate, openDbClient, safeInitDbClient, strictInitDbClientFromFilePath };

package/dist/client.js

@@ -6,7 +6,7 @@ import {
 import invariant from "tiny-invariant";
 
 // src/migration.ts
-var migrate1 = `
+var initial = `
 CREATE TABLE IF NOT EXISTS file_tasks (
     id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
     path TEXT NOT NULL UNIQUE,
@@ -87,7 +87,28 @@ CREATE INDEX timings_file_task_id_idx ON timings(file_task_id);
 CREATE INDEX timings_task_run_id_idx ON timings(task_run_id);
 CREATE INDEX timings_created_at_idx ON timings(created_at);
 `;
-var migration = [migrate1];
+var migrations = [
+  { id: "0001-initial", up: initial },
+  // What `task` returned, as JSON. NULL for a run that returned nothing or never finished.
+  { id: "0002-task-run-result", up: "ALTER TABLE task_runs ADD COLUMN result TEXT" },
+  // Named leases held by runs (ctx.lock, `exclusive`). A row past expires_at is free to take.
+  {
+    id: "0003-locks",
+    up: `CREATE TABLE IF NOT EXISTS locks (
+      name TEXT PRIMARY KEY NOT NULL,
+      holder TEXT NOT NULL,
+      task_run_id INTEGER,
+      acquired_at INTEGER NOT NULL,
+      expires_at INTEGER NOT NULL
+    )`
+  },
+  // A retry points at the run it retries (the first of its tick); a skip says why.
+  {
+    id: "0004-task-run-retry-and-skip-reason",
+    up: `ALTER TABLE task_runs ADD COLUMN retry_of INTEGER REFERENCES task_runs(id) ON DELETE CASCADE;
+    ALTER TABLE task_runs ADD COLUMN skip_reason TEXT`
+  }
+];
 
 // src/client.ts
 var BUSY_TIMEOUT_MS = 5e3;
@@ -117,10 +138,59 @@ function normalizeDbPath(dbPath) {
   const normalizedFilePath = dbPath.startsWith("file:") ? dbPath : `file:${dbPath}`;
   return normalizedFilePath;
 }
-async function runMigrations(dbClient) {
+var MIGRATIONS_TABLE = "ngn_migrations";
+function splitStatements(sql) {
+  return sql.split(";").map((stmt) => stmt.trim()).filter((stmt) => stmt.length > 0);
+}
+async function tableExists(dbClient, name) {
+  const { rows } = await dbClient.execute({
+    sql: "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?",
+    args: [name]
+  });
+  return rows.length > 0;
+}
+async function appliedMigrationIds(dbClient) {
+  if (await tableExists(dbClient, MIGRATIONS_TABLE)) {
+    const { rows } = await dbClient.execute(`SELECT id FROM ${MIGRATIONS_TABLE}`);
+    return new Set(rows.map((row) => row.id));
+  }
+  return new Set(await tableExists(dbClient, "file_tasks") ? [migrations[0].id] : []);
+}
+function migrationStatements(pending, applied) {
+  const statements = [
+    `CREATE TABLE IF NOT EXISTS ${MIGRATIONS_TABLE} (
+      id TEXT PRIMARY KEY NOT NULL,
+      applied_at INTEGER NOT NULL DEFAULT(unixepoch('subsecond') * 1000)
+    )`
+  ];
+  for (const id of applied) {
+    statements.push({
+      sql: `INSERT OR IGNORE INTO ${MIGRATIONS_TABLE} (id) VALUES (?)`,
+      args: [id]
+    });
+  }
+  for (const migration of pending) {
+    statements.push(...splitStatements(migration.up));
+    statements.push({
+      sql: `INSERT INTO ${MIGRATIONS_TABLE} (id) VALUES (?)`,
+      args: [migration.id]
+    });
+  }
+  return statements;
+}
+async function migrate(dbClient) {
   invariant(dbClient, "dbClient is not initialized");
-  const migrationCommands = migration[0].split(";").map((stmt) => stmt.trim()).filter((stmt) => stmt.length > 0);
-  return dbClient.batch(migrationCommands, "write");
+  for (let attempt = 0; ; attempt++) {
+    const applied = await appliedMigrationIds(dbClient);
+    const pending = migrations.filter((migration) => !applied.has(migration.id));
+    if (pending.length === 0 && await tableExists(dbClient, MIGRATIONS_TABLE)) return [];
+    try {
+      await dbClient.batch(migrationStatements(pending, applied), "write");
+      return pending.map((migration) => migration.id);
+    } catch (error) {
+      if (attempt > 0) throw error;
+    }
+  }
 }
 async function initDbFileIfNotExists(dbPath) {
   invariant(dbPath, "dbPath is required");
@@ -129,7 +199,7 @@ async function initDbFileIfNotExists(dbPath) {
   const dbClient = openDbClient(normalizedDbPath);
   try {
     if (!dbFileExists) {
-      await runMigrations(dbClient);
+      await migrate(dbClient);
     }
   } finally {
     dbClient.close();
@@ -152,11 +222,11 @@ async function safeInitDbClient(sqliteUrl) {
   let dbClient;
   if (sqliteUrl === ":memory:") {
     dbClient = openDbClient(sqliteUrl, { wal: false });
-    await runMigrations(dbClient);
   } else {
     const normalizedDbPath = await initDbFileIfNotExists(sqliteUrl);
     dbClient = openDbClient(normalizedDbPath);
   }
+  await migrate(dbClient);
   return dbClient;
 }
 function isDbPath(dbPath) {
@@ -168,6 +238,7 @@ export {
   createClient,
   initDbFileIfNotExists,
   isDbPath,
+  migrate,
   openDbClient,
   safeInitDbClient,
   strictInitDbClientFromFilePath

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@apisurf/ngn-persistence",
-  "version": "0.2.7",
+  "version": "0.2.8",
   "description": "ngn SQLite persistence: connection pragmas, migrations and the client.",
   "type": "module",
   "author": "Luka Vidakovic",
@@ -37,14 +37,16 @@
     "@libsql/client": "^0.14.0",
     "tiny-invariant": "^1.3.3",
     "zod": "^3.24.2",
-    "@apisurf/ngn-os": "0.2.7"
+    "@apisurf/ngn-os": "0.2.8"
   },
   "devDependencies": {
     "@types/node": "^22.8.1",
-    "tsup": "^8.3.5"
+    "tsup": "^8.3.5",
+    "vitest": "4.1.8"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
+    "test": "vitest run",
     "build": "tsup src/client.ts --dts --out-dir dist --format esm",
     "dev": "tsup src/client.ts --dts --out-dir dist --format esm --watch"
   }