@happyvertical/smrt-core 0.40.69 → 0.41.0

package/dist/migrations/tracker.js

@@ -1,5 +1,6 @@
 import { detectEngine } from "../schema/ddl/index.js";
 import { CREATE_SMRT_SCHEMA_MIGRATIONS_TABLE, getSystemTableDDLForEngine } from "../system/schema.js";
+import { parsePostgresTimeoutMs } from "../postgres-timeouts.js";
 import { assertPostgresSystemTimestampsCurrent } from "../system/compatibility.js";
 import { computeChecksum, verifyChecksum } from "./checksum.js";
 import { createLogger } from "@happyvertical/logger";
@@ -31,6 +32,16 @@ var AtomicMigrationRollback = class extends Error {
 	}
 };
 var CONCURRENT_INDEX_STATEMENT_RE = /(?:(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX)\s+CONCURRENTLY|REINDEX(?:\s*\([^)]*\))?\s+(?:INDEX|TABLE|SCHEMA|DATABASE|SYSTEM)\s+CONCURRENTLY)/i;
+/**
+* Matches the leading clause of a `CREATE [UNIQUE] INDEX` statement and
+* captures the index name, whether it is written bare or double-quoted, and
+* regardless of the optional `CONCURRENTLY` / `IF NOT EXISTS` clauses.
+*
+* Used by the PostgreSQL concurrent-index mode to correlate a pending create
+* with an INVALID index left behind by an earlier failed
+* `CREATE INDEX CONCURRENTLY` (issue #2362).
+*/
+var CREATE_INDEX_NAME_RE = /^\s*CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:CONCURRENTLY\s+)?(?:IF\s+NOT\s+EXISTS\s+)?(?:"((?:[^"]|"")+)"|([A-Za-z_][\w$]*))/i;
 function findConcurrentIndexStatement(definitions) {
 	for (const definition of definitions) {
 		const statement = definition.up.find((sql) => CONCURRENT_INDEX_STATEMENT_RE.test(sql));
@@ -42,6 +53,61 @@ function findConcurrentIndexStatement(definitions) {
 	return null;
 }
 /**
+* Extract the index name from a `CREATE [UNIQUE] INDEX …` statement.
+*
+* Returns the unquoted identifier, or `null` when the statement is not a
+* CREATE INDEX (a drop, a reindex, or unrelated DDL).
+*
+* Exported for unit testing and because the CLI's index rollout needs the
+* same correlation to explain what it repaired.
+*/
+function extractCreatedIndexName(sql) {
+	const match = CREATE_INDEX_NAME_RE.exec(sql);
+	if (!match) return null;
+	return match[1] !== void 0 ? match[1].replace(/""/g, "\"") : match[2];
+}
+/**
+* Coerce a caller-supplied timeout to a usable millisecond count.
+*
+* Applied once, in the constructor, so every later read of
+* `this.options.lockTimeout` / `.statementTimeout` is already finite and
+* non-negative. Each timeout falls back to *its own* default — a bad
+* `lockTimeout` must not silently inherit the (much larger) statement-timeout
+* default and weaken the lock bound it was meant to impose.
+*/
+function resolveTimeoutOption(value, fallback) {
+	return typeof value === "number" && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : fallback;
+}
+/**
+* Render a millisecond timeout as a PostgreSQL interval literal.
+*
+* PostgreSQL reads `lock_timeout`/`statement_timeout` as milliseconds when
+* given a bare number, but the explicit `ms` unit keeps the emitted SQL
+* self-documenting in migration logs. `0` disables the timeout, which is
+* PostgreSQL's own semantic and therefore preserved verbatim.
+*
+* Input is already normalized by {@link resolveTimeoutOption}.
+*/
+function formatPostgresTimeout(milliseconds) {
+	return `${Math.max(0, Math.trunc(milliseconds))}ms`;
+}
+/**
+* Prefix stamped on the `error_message` of every concurrent-index migration
+* whose transactional half has already committed.
+*
+* This is the resume marker: it is the only durable record that phase 1's
+* statements ran, so a retry can skip straight to the index build. It must
+* therefore be carried by *both* the interim write at the end of phase 1 and
+* every failure recorded during phase 2 — a lock timeout on the index build
+* leaves the non-index DDL committed just as surely as a crash does.
+*/
+var CONCURRENT_INDEX_PHASE1_MARKER = "[smrt: concurrent-index phase 1 committed]";
+/**
+* Interim `error_message` for a migration whose transactional half has
+* committed but whose `CONCURRENTLY` index build has not run yet.
+*/
+var CONCURRENT_INDEX_PENDING_MESSAGE = `${CONCURRENT_INDEX_PHASE1_MARKER} Concurrent index build pending: the non-index half of this migration committed but the CONCURRENTLY index DDL had not completed. Re-run db:migrate to finish it.`;
+/**
 * MigrationTracker class
 *
 * Provides migration tracking and state management:
@@ -66,11 +132,20 @@ var MigrationTracker = class MigrationTracker {
 	*/
 	initializePromise = null;
 	currentBatch = null;
+	/**
+	* Statements deferred out of the current atomic batch by PostgreSQL
+	* concurrent-index mode. The parent tracker sets this on the
+	* transaction-scoped tracker before phase 1 so `apply()` executes only the
+	* transaction-safe statements and leaves the migration record in `running`;
+	* the parent finalizes those records after the transaction commits and the
+	* `CONCURRENTLY` DDL has run. `null` on every ordinary path.
+	*/
+	concurrentIndexPlan = null;
 	constructor(options) {
 		this.db = options.db;
 		this.options = {
-			lockTimeout: options.lockTimeout ?? DEFAULT_OPTIONS.lockTimeout,
-			statementTimeout: options.statementTimeout ?? DEFAULT_OPTIONS.statementTimeout,
+			lockTimeout: resolveTimeoutOption(options.lockTimeout, DEFAULT_OPTIONS.lockTimeout),
+			statementTimeout: resolveTimeoutOption(options.statementTimeout, DEFAULT_OPTIONS.statementTimeout),
 			useConcurrentIndexes: options.useConcurrentIndexes ?? DEFAULT_OPTIONS.useConcurrentIndexes
 		};
 		this.engineHint = options.engineHint;
@@ -263,9 +338,25 @@ var MigrationTracker = class MigrationTracker {
 		else await this.db.query(`INSERT INTO _smrt_schema_migrations
          (id, name, version, checksum, status, attempts, is_reversible, package_name, source_file, applied_by, batch)
          VALUES (?, ?, ?, ?, 'running', 1, ?, ?, ?, ?, ?)`, id, definition.id, definition.version, checksum, definition.is_reversible ?? definition.down.length > 0 ? 1 : 0, definition.package_name ?? null, definition.source_file ?? null, hostname(), this.currentBatch);
+		const split = this.concurrentIndexPlan?.get(definition.id);
+		const deferredConcurrent = (split?.concurrent.length ?? 0) > 0;
+		const phase1AlreadyCommitted = deferredConcurrent && existing?.status === "failed" && existing.checksum === checksum && typeof existing.error_message === "string" && existing.error_message.startsWith(CONCURRENT_INDEX_PHASE1_MARKER);
 		try {
-			await this.executeStatements(definition.up, options);
+			await this.executeStatements(phase1AlreadyCommitted ? [] : split ? split.regular : definition.up, options);
 			const executionTime = Date.now() - startTime;
+			if (deferredConcurrent) {
+				await this.db.query(`UPDATE _smrt_schema_migrations
+           SET status = 'failed', execution_time_ms = ?, error_message = ?
+           WHERE id = ?`, executionTime, CONCURRENT_INDEX_PENDING_MESSAGE, id);
+				return {
+					success: true,
+					applied: true,
+					skipped: false,
+					name: definition.id,
+					checksum,
+					execution_time_ms: executionTime
+				};
+			}
 			await this.db.query(`UPDATE _smrt_schema_migrations
          SET status = 'completed', execution_time_ms = ?, applied_checksum = ?, applied_at = CURRENT_TIMESTAMP
          WHERE id = ?`, executionTime, checksum, id);
@@ -299,36 +390,66 @@ var MigrationTracker = class MigrationTracker {
 		}
 	}
 	/**
-	* Apply multiple migrations in order
+	* Apply multiple migrations in order.
+	*
+	* On PostgreSQL the atomic path always bounds itself with
+	* `SET LOCAL lock_timeout` / `SET LOCAL statement_timeout` so a migration
+	* that queues behind a long-running writer fails fast and rolls back
+	* instead of stalling every writer behind its already-held locks
+	* (issue #2362).
+	*
+	* `options.postgresSafe` additionally enables **concurrent-index mode**:
+	* non-index DDL still runs in the one atomic transaction, then every index
+	* statement runs `CONCURRENTLY` on a pinned session after that transaction
+	* commits. That mode trades atomicity for availability — a failure part way
+	* through the index phase leaves the committed non-index DDL in place, and
+	* the unfinished index migrations are recorded `failed` so the next run
+	* retries them, resuming at the index phase rather than re-running the
+	* non-index statements that already committed.
 	*/
 	async applyAll(definitions, options = {}) {
 		if (options.atomic && !options.dryRun) {
 			if (!this.db.transaction) throw new Error("Atomic migration batches require a database adapter with transaction support.");
-			if (this.dbEngine === "postgres") {
+			const isPostgres = this.dbEngine === "postgres";
+			const concurrentIndexMode = isPostgres && Boolean(options.postgresSafe);
+			const plan = concurrentIndexMode ? buildConcurrentIndexPlan(definitions, this.options.useConcurrentIndexes) : null;
+			if (isPostgres && !concurrentIndexMode) {
 				const concurrentStatement = findConcurrentIndexStatement(definitions);
-				if (concurrentStatement) throw new Error(`Atomic migration batch cannot include CONCURRENTLY index DDL in ${concurrentStatement.migrationName}; PostgreSQL forbids CONCURRENTLY inside a transaction.`);
+				if (concurrentStatement) throw new Error(`Atomic migration batch cannot include CONCURRENTLY index DDL in ${concurrentStatement.migrationName}; PostgreSQL forbids CONCURRENTLY inside a transaction. Re-run with PostgreSQL concurrent-index mode (smrt db:migrate --postgres-safe) to build those indexes outside the batch.`);
 			}
+			const deferred = plan && plan.size > 0 ? plan : null;
 			try {
-				return await this.db.transaction(async (tx) => {
+				const committed = await this.db.transaction(async (tx) => {
 					const txDb = {
 						...tx,
 						transaction: async (callback) => callback(tx)
 					};
-					const results = await new MigrationTracker({
+					if (isPostgres) {
+						await tx.query(`SET LOCAL lock_timeout = '${formatPostgresTimeout(this.options.lockTimeout)}'`);
+						await tx.query(`SET LOCAL statement_timeout = '${formatPostgresTimeout(this.options.statementTimeout)}'`);
+					}
+					const txTracker = new MigrationTracker({
 						db: txDb,
 						lockTimeout: this.options.lockTimeout,
 						statementTimeout: this.options.statementTimeout,
 						useConcurrentIndexes: false
-					}).applyAll(definitions, {
+					});
+					txTracker.concurrentIndexPlan = deferred;
+					const results = await txTracker.applyAll(definitions, {
 						...options,
 						atomic: false,
 						continueOnError: false,
-						postgresSafe: false
+						postgresSafe: false,
+						onProgress: options.onProgress ? (result) => {
+							if (!deferred?.has(result.name)) options.onProgress?.(result);
+						} : void 0
 					});
 					const failed = results.find((result) => !result.success);
 					if (failed) throw new AtomicMigrationRollback(results, failed);
 					return results;
 				});
+				if (!deferred) return committed;
+				return await this.applyDeferredConcurrentIndexes(committed, deferred, options);
 			} catch (error) {
 				if (error instanceof AtomicMigrationRollback) return error.results.map((result) => {
 					if (!result.success || !result.applied) return result;
@@ -356,6 +477,161 @@ var MigrationTracker = class MigrationTracker {
 		return results;
 	}
 	/**
+	* Phase 2 of PostgreSQL concurrent-index mode.
+	*
+	* Runs every deferred index statement on one pinned session, outside the
+	* committed transaction, bounded by session `lock_timeout` /
+	* `statement_timeout`. INVALID indexes left behind by an earlier failed
+	* `CREATE INDEX CONCURRENTLY` are dropped first so the retry rebuilds them
+	* rather than skipping over a stump that reads as present.
+	*
+	* On failure the batch stops: the failing migration and every migration it
+	* did not reach are recorded `failed`, which the CLI's reconciling
+	* `db:migrate` retries on the next run.
+	*/
+	async applyDeferredConcurrentIndexes(committed, plan, options) {
+		const results = [...committed];
+		const session = await this.acquireMigrationSession();
+		try {
+			await session.query(`SET lock_timeout = '${formatPostgresTimeout(this.options.lockTimeout)}'`);
+			await session.query(`SET statement_timeout = '${formatPostgresTimeout(this.options.statementTimeout)}'`);
+			const invalidIndexes = await this.readInvalidIndexNames(session);
+			let stoppedBy = null;
+			for (let index = 0; index < results.length; index++) {
+				const result = results[index];
+				const split = plan.get(result.name);
+				if (!split || split.concurrent.length === 0) continue;
+				if (result.skipped) continue;
+				if (stoppedBy) {
+					const error = /* @__PURE__ */ new Error(`Not attempted: concurrent index phase stopped after ${stoppedBy} failed`);
+					await this.recordConcurrentIndexFailure(result.name, error.message);
+					results[index] = {
+						...result,
+						success: false,
+						applied: false,
+						skipped: false,
+						error
+					};
+					options.onProgress?.(results[index]);
+					continue;
+				}
+				const startedAt = Date.now();
+				try {
+					for (const sql of split.concurrent) {
+						const indexName = extractCreatedIndexName(sql);
+						if (indexName && invalidIndexes.has(indexName)) {
+							logger.warn(`Rebuilding INVALID PostgreSQL index ${indexName} left by a previous failed CREATE INDEX CONCURRENTLY`);
+							await session.query(`DROP INDEX CONCURRENTLY IF EXISTS ${quoteIndexName(indexName)}`);
+							invalidIndexes.delete(indexName);
+						}
+						await session.query(sql);
+					}
+					const executionTime = Math.max(1, (result.execution_time_ms ?? 0) + (Date.now() - startedAt));
+					await this.recordConcurrentIndexSuccess(result.name, result.checksum, executionTime);
+					results[index] = {
+						...result,
+						success: true,
+						applied: true,
+						skipped: false,
+						execution_time_ms: executionTime
+					};
+				} catch (error) {
+					const indexError = error instanceof Error ? error : new Error(String(error));
+					if (!options.continueOnError) stoppedBy = result.name;
+					await this.recordConcurrentIndexFailure(result.name, indexError.message);
+					results[index] = {
+						...result,
+						success: false,
+						applied: false,
+						skipped: false,
+						execution_time_ms: Date.now() - startedAt,
+						error: indexError
+					};
+				}
+				options.onProgress?.(results[index]);
+			}
+			return results;
+		} finally {
+			await session.release();
+		}
+	}
+	/**
+	* Pin one connection for the concurrent-index phase.
+	*
+	* PostgreSQL pools hand each top-level `query` an arbitrary connection, so
+	* a `SET lock_timeout` issued through the pool may not apply to the DDL
+	* that follows. Adapters that expose `acquireSession` give us a pinned
+	* connection; anything else (SQLite, DuckDB, test doubles) is
+	* single-connection anyway and falls back to the plain interface.
+	*/
+	async acquireMigrationSession() {
+		if (typeof this.db.acquireSession === "function") {
+			const session = await this.db.acquireSession();
+			return {
+				query: (sql, ...vars) => session.query(sql, ...vars),
+				release: () => session.release()
+			};
+		}
+		return {
+			query: (sql, ...vars) => this.db.query(sql, ...vars),
+			release: async () => {}
+		};
+	}
+	/**
+	* Read the names of INVALID indexes that an unqualified index name in this
+	* session would actually resolve to.
+	*
+	* `pg_indexes` (what the SQL adapter's introspection reads) lists an
+	* INVALID index exactly like a healthy one, so the schema differ believes a
+	* failed `CREATE INDEX CONCURRENTLY` succeeded. `pg_index.indisvalid` is the
+	* authoritative flag.
+	*
+	* Scoped to `current_schemas(false)` — the session's search path — because
+	* migration DDL names indexes unqualified, and so does the
+	* `DROP INDEX CONCURRENTLY IF EXISTS "<name>"` repair. Scanning every schema
+	* would let an unrelated invalid index of the same name in another schema
+	* trigger a drop that then resolves to a *different* object. Where the same
+	* name appears in several search-path schemas, the earliest on the path wins
+	* — the same rule PostgreSQL uses to resolve the name.
+	*
+	* A role without catalog access degrades to "no known invalid indexes"
+	* rather than aborting the migration.
+	*/
+	async readInvalidIndexNames(session) {
+		const names = /* @__PURE__ */ new Set();
+		try {
+			const result = await session.query(`SELECT DISTINCT ON (c.relname) c.relname AS index_name
+           FROM pg_index i
+           JOIN pg_class c ON c.oid = i.indexrelid
+           JOIN pg_namespace n ON n.oid = c.relnamespace
+          WHERE i.indisvalid = false
+            AND n.nspname = ANY (current_schemas(false))
+          ORDER BY c.relname,
+                   array_position(current_schemas(false), n.nspname::text)`);
+			for (const row of result.rows) {
+				const name = row.index_name;
+				if (typeof name === "string") names.add(name);
+			}
+		} catch (error) {
+			logger.warn(`Could not read pg_index.indisvalid; skipping INVALID index repair: ${error instanceof Error ? error.message : String(error)}`);
+		}
+		return names;
+	}
+	async recordConcurrentIndexSuccess(name, checksum, executionTime) {
+		await this.db.query(`UPDATE _smrt_schema_migrations
+         SET status = 'completed', execution_time_ms = ?, applied_checksum = ?, error_message = NULL, applied_at = CURRENT_TIMESTAMP
+       WHERE name = ?`, executionTime, checksum, name);
+	}
+	async recordConcurrentIndexFailure(name, message) {
+		try {
+			await this.db.query(`UPDATE _smrt_schema_migrations
+           SET status = 'failed', error_message = ?
+         WHERE name = ?`, `${CONCURRENT_INDEX_PHASE1_MARKER} ${message}`, name);
+		} catch (persistError) {
+			logger.error(`Failed to persist failed status for migration ${name}: ${persistError instanceof Error ? persistError.message : String(persistError)}`);
+		}
+	}
+	/**
 	* Rollback a migration
 	*/
 	async rollback(name, definition, options = {}) {
@@ -472,18 +748,57 @@ var MigrationTracker = class MigrationTracker {
 	async executePostgresStatements(statements) {
 		const { concurrent: finalConcurrent, regular: finalRegular } = planPostgresStatements(statements, !!this.options.useConcurrentIndexes);
 		if (finalRegular.length > 0 && this.db.transaction) await this.db.transaction(async (tx) => {
-			await tx.query(`SET LOCAL lock_timeout = '${this.options.lockTimeout}ms'`);
-			await tx.query(`SET LOCAL statement_timeout = '${this.options.statementTimeout}ms'`);
+			await tx.query(`SET LOCAL lock_timeout = '${formatPostgresTimeout(this.options.lockTimeout)}'`);
+			await tx.query(`SET LOCAL statement_timeout = '${formatPostgresTimeout(this.options.statementTimeout)}'`);
 			for (const sql of finalRegular) await tx.query(sql);
 		});
 		else if (finalRegular.length > 0) for (const sql of finalRegular) await this.db.query(sql);
-		for (const sql of finalConcurrent) {
-			await this.db.query(`SET lock_timeout = '${this.options.lockTimeout}ms'`);
-			await this.db.query(sql);
+		if (finalConcurrent.length === 0) return;
+		const session = await this.acquireMigrationSession();
+		try {
+			await session.query(`SET lock_timeout = '${formatPostgresTimeout(this.options.lockTimeout)}'`);
+			await session.query(`SET statement_timeout = '${formatPostgresTimeout(this.options.statementTimeout)}'`);
+			const invalidIndexes = await this.readInvalidIndexNames(session);
+			for (const sql of finalConcurrent) {
+				const indexName = extractCreatedIndexName(sql);
+				if (indexName && invalidIndexes.has(indexName)) {
+					logger.warn(`Rebuilding INVALID PostgreSQL index ${indexName} left by a previous failed CREATE INDEX CONCURRENTLY`);
+					await session.query(`DROP INDEX CONCURRENTLY IF EXISTS ${quoteIndexName(indexName)}`);
+					invalidIndexes.delete(indexName);
+				}
+				await session.query(sql);
+			}
+		} finally {
+			await session.release();
 		}
 	}
 };
 /**
+* Quote an index identifier for PostgreSQL, escaping embedded double quotes.
+*/
+function quoteIndexName(name) {
+	return `"${name.replace(/"/g, "\"\"")}"`;
+}
+/**
+* Build the PostgreSQL concurrent-index plan for an atomic batch.
+*
+* Returns one entry per migration definition that has at least one statement
+* which must leave the transaction, mapping it to the transaction-safe
+* statements (phase 1) and the `CONCURRENTLY` statements (phase 2).
+* Definitions with nothing to defer are omitted so the caller can treat
+* `plan.has(id)` as "this migration finishes in phase 2".
+*
+* Exported for unit testing.
+*/
+function buildConcurrentIndexPlan(definitions, useConcurrentIndexes) {
+	const plan = /* @__PURE__ */ new Map();
+	for (const definition of definitions) {
+		const split = planPostgresStatements(definition.up, useConcurrentIndexes);
+		if (split.concurrent.length > 0) plan.set(definition.id, split);
+	}
+	return plan;
+}
+/**
 * Split a batch of SQL statements into a transaction-safe set and a
 * concurrent set, applying `--postgres-safe` rewriting if requested.
 *
@@ -527,6 +842,6 @@ function planPostgresStatements(statements, useConcurrentIndexes) {
 	};
 }
 //#endregion
-export { MigrationTracker, planPostgresStatements };
+export { MigrationTracker, buildConcurrentIndexPlan, extractCreatedIndexName, parsePostgresTimeoutMs, planPostgresStatements };
 
 //# sourceMappingURL=tracker.js.map