@happyvertical/smrt-core 0.30.0

package/dist/migrations/tracker.js

@@ -0,0 +1,624 @@
+import { randomUUID } from "node:crypto";
+import { hostname } from "node:os";
+import { createLogger } from "@happyvertical/logger";
+import { detectEngine } from "../schema/ddl/index.js";
+import { CREATE_SMRT_SCHEMA_MIGRATIONS_TABLE } from "../system/schema.js";
+import { computeChecksum, verifyChecksum } from "./checksum.js";
+const logger = createLogger({ level: "info" });
+const DEFAULT_OPTIONS = {
+  lockTimeout: 3e4,
+  statementTimeout: 6e4,
+  useConcurrentIndexes: true
+};
+class AtomicMigrationRollback extends Error {
+  constructor(results, failed) {
+    super(`Atomic migration batch failed at ${failed.name}`);
+    this.results = results;
+    this.failed = failed;
+  }
+  results;
+  failed;
+}
+const CONCURRENT_INDEX_STATEMENT_RE = /(?:(?:CREATE\s+(?:UNIQUE\s+)?INDEX|DROP\s+INDEX)\s+CONCURRENTLY|REINDEX(?:\s*\([^)]*\))?\s+(?:INDEX|TABLE|SCHEMA|DATABASE|SYSTEM)\s+CONCURRENTLY)/i;
+function findConcurrentIndexStatement(definitions) {
+  for (const definition of definitions) {
+    const statement = definition.up.find(
+      (sql) => CONCURRENT_INDEX_STATEMENT_RE.test(sql)
+    );
+    if (statement) {
+      return {
+        migrationName: definition.id,
+        statement
+      };
+    }
+  }
+  return null;
+}
+class MigrationTracker {
+  db;
+  options;
+  engineHint;
+  dbEngine;
+  /**
+   * Memoized initialization promise. Storing the in-flight promise (rather
+   * than a `boolean` flag set after the DDL completes) makes `initialize()`
+   * safe under concurrency: multiple parallel callers all `await` the same
+   * promise instead of independently re-running the DDL. On error the slot
+   * is cleared so the next caller retries — a transient failure doesn't
+   * permanently poison the instance.
+   */
+  initializePromise = null;
+  currentBatch = null;
+  constructor(options) {
+    this.db = options.db;
+    this.options = {
+      lockTimeout: options.lockTimeout ?? DEFAULT_OPTIONS.lockTimeout,
+      statementTimeout: options.statementTimeout ?? DEFAULT_OPTIONS.statementTimeout,
+      useConcurrentIndexes: options.useConcurrentIndexes ?? DEFAULT_OPTIONS.useConcurrentIndexes
+    };
+    this.engineHint = options.engineHint;
+    this.dbEngine = this.detectDbEngine();
+  }
+  /**
+   * Detect the database engine. Honors an explicit `engineHint` (passed via
+   * `MigrationTrackerOptions`) before falling back to URL parsing — this is
+   * the same `detectEngine` contract used by `SchemaComparer` and the rest
+   * of the migration stack, so the tracker stays consistent with the DDL
+   * the orchestrator generated.
+   */
+  detectDbEngine() {
+    const dbWithConfig = this.db;
+    const url = this.db.url || dbWithConfig.config?.url || "";
+    return detectEngine(url, this.engineHint);
+  }
+  /**
+   * Get the current database engine
+   */
+  getEngine() {
+    return this.dbEngine;
+  }
+  /**
+   * Initialize the migrations tracking table
+   */
+  async initialize() {
+    if (!this.initializePromise) {
+      this.initializePromise = this.runInitializeDdl().catch((error) => {
+        this.initializePromise = null;
+        throw error;
+      });
+    }
+    return this.initializePromise;
+  }
+  async runInitializeDdl() {
+    const statements = CREATE_SMRT_SCHEMA_MIGRATIONS_TABLE.split(";").map((s) => s.trim()).filter((s) => s.length > 0);
+    for (const statement of statements) {
+      await this.db.query(statement);
+    }
+  }
+  /**
+   * Get all applied migrations ordered by applied_at
+   */
+  async getAppliedMigrations() {
+    await this.initialize();
+    const result = await this.db.query(
+      `SELECT * FROM _smrt_schema_migrations WHERE status = 'completed' ORDER BY applied_at ASC`
+    );
+    return result.rows.map((row) => ({
+      ...row,
+      applied_at: new Date(row.applied_at),
+      rolled_back_at: row.rolled_back_at ? new Date(row.rolled_back_at) : null,
+      is_reversible: Boolean(row.is_reversible)
+    }));
+  }
+  /**
+   * Get pending migrations (defined but not applied)
+   */
+  async getPendingMigrations(definitions) {
+    const applied = await this.getAppliedMigrations();
+    const appliedNames = new Set(applied.map((m) => m.name));
+    return definitions.filter((d) => !appliedNames.has(d.id));
+  }
+  /**
+   * Check if a migration has been applied
+   */
+  async isApplied(migrationId) {
+    await this.initialize();
+    const result = await this.db.query(
+      `SELECT 1 FROM _smrt_schema_migrations WHERE name = ? AND status = 'completed' LIMIT 1`,
+      migrationId
+    );
+    return result.rows.length > 0;
+  }
+  /**
+   * Get a single migration record by name
+   */
+  async getMigration(name) {
+    await this.initialize();
+    const result = await this.db.query(
+      `SELECT * FROM _smrt_schema_migrations WHERE name = ? LIMIT 1`,
+      name
+    );
+    if (result.rows.length === 0) return null;
+    const row = result.rows[0];
+    return {
+      ...row,
+      applied_at: new Date(row.applied_at),
+      rolled_back_at: row.rolled_back_at ? new Date(row.rolled_back_at) : null,
+      is_reversible: Boolean(row.is_reversible)
+    };
+  }
+  /**
+   * Get the next batch number
+   */
+  async getNextBatch() {
+    await this.initialize();
+    const result = await this.db.query(
+      `SELECT MAX(batch) as max_batch FROM _smrt_schema_migrations`
+    );
+    const row = result.rows[0];
+    return (row?.max_batch ?? 0) + 1;
+  }
+  /**
+   * Detect drift between migration definitions and applied migrations
+   */
+  async detectDrift(definitions) {
+    const applied = await this.getAppliedMigrations();
+    const reports = [];
+    new Map(applied.map((m) => [m.name, m]));
+    const definitionMap = new Map(definitions.map((d) => [d.id, d]));
+    for (const record of applied) {
+      const definition = definitionMap.get(record.name);
+      if (!definition) {
+        reports.push({
+          migration_name: record.name,
+          expected_checksum: "N/A",
+          actual_checksum: record.checksum,
+          drift_type: "unknown_in_db",
+          recommendation: "Migration exists in database but definition is missing. Manual cleanup may be required."
+        });
+        continue;
+      }
+      const currentChecksum = computeChecksum(definition.up);
+      const verification = verifyChecksum(
+        currentChecksum,
+        record.checksum,
+        record.applied_checksum
+      );
+      if (!verification.valid) {
+        const driftType = verification.drift ?? "db_modified";
+        reports.push({
+          migration_name: record.name,
+          expected_checksum: record.checksum,
+          actual_checksum: currentChecksum,
+          drift_type: driftType,
+          recommendation: driftType === "file_modified" ? "Migration file was modified after application. Create a new migration instead." : "Database schema was modified outside of migrations. Review and reconcile manually."
+        });
+      }
+    }
+    return reports;
+  }
+  /**
+   * Apply a single migration
+   */
+  async apply(definition, options = {}) {
+    await this.initialize();
+    const startTime = Date.now();
+    const checksum = computeChecksum(definition.up);
+    const existing = await this.getMigration(definition.id);
+    if (existing) {
+      switch (existing.status) {
+        case "completed":
+          if (!options.force && existing.checksum !== checksum) {
+            return {
+              success: false,
+              applied: false,
+              skipped: false,
+              name: definition.id,
+              checksum,
+              execution_time_ms: 0,
+              error: `Migration ${definition.id} checksum mismatch. Was already applied with different checksum. Use --force to override.`
+            };
+          }
+          if (!options.force && !options.reconcile) {
+            return {
+              success: true,
+              applied: false,
+              skipped: true,
+              name: definition.id,
+              checksum,
+              execution_time_ms: 0
+            };
+          }
+          break;
+        case "failed":
+          if (!options.force && !options.reconcile) {
+            return {
+              success: false,
+              applied: false,
+              skipped: false,
+              name: definition.id,
+              checksum,
+              execution_time_ms: 0,
+              error: `Migration ${definition.id} previously failed: ${existing.error_message || "Unknown error"}. Use --force to retry.`
+            };
+          }
+          break;
+        case "running":
+          if (!options.force) {
+            return {
+              success: false,
+              applied: false,
+              skipped: false,
+              name: definition.id,
+              checksum,
+              execution_time_ms: 0,
+              error: `Migration ${definition.id} is currently running or was interrupted. Use --force to retry if the previous run crashed.`
+            };
+          }
+          break;
+      }
+    }
+    if (options.dryRun) {
+      return {
+        success: true,
+        applied: false,
+        skipped: false,
+        name: definition.id,
+        checksum,
+        execution_time_ms: Date.now() - startTime
+      };
+    }
+    if (this.currentBatch === null) {
+      this.currentBatch = await this.getNextBatch();
+    }
+    const id = existing?.id || randomUUID();
+    const attempts = (existing?.attempts || 0) + 1;
+    if (existing) {
+      await this.db.query(
+        `UPDATE _smrt_schema_migrations
+         SET status = 'running', checksum = ?, attempts = ?, error_message = NULL, batch = ?, applied_by = ?, applied_at = CURRENT_TIMESTAMP
+         WHERE id = ?`,
+        checksum,
+        attempts,
+        this.currentBatch,
+        hostname(),
+        id
+      );
+    } else {
+      await this.db.query(
+        `INSERT INTO _smrt_schema_migrations
+         (id, name, version, checksum, status, attempts, is_reversible, package_name, source_file, applied_by, batch)
+         VALUES (?, ?, ?, ?, 'running', 1, ?, ?, ?, ?, ?)`,
+        id,
+        definition.id,
+        definition.version,
+        checksum,
+        definition.is_reversible ?? definition.down.length > 0 ? 1 : 0,
+        definition.package_name ?? null,
+        definition.source_file ?? null,
+        hostname(),
+        this.currentBatch
+      );
+    }
+    try {
+      await this.executeStatements(definition.up, options);
+      const executionTime = Date.now() - startTime;
+      await this.db.query(
+        `UPDATE _smrt_schema_migrations
+         SET status = 'completed', execution_time_ms = ?, applied_checksum = ?, applied_at = CURRENT_TIMESTAMP
+         WHERE id = ?`,
+        executionTime,
+        checksum,
+        id
+      );
+      return {
+        success: true,
+        applied: true,
+        skipped: false,
+        name: definition.id,
+        checksum,
+        execution_time_ms: executionTime
+      };
+    } catch (error) {
+      const migrationError = error instanceof Error ? error : new Error(String(error));
+      try {
+        await this.db.query(
+          `UPDATE _smrt_schema_migrations
+           SET status = 'failed', error_message = ?
+           WHERE id = ?`,
+          migrationError.message,
+          id
+        );
+      } catch (persistError) {
+        const persistMessage = persistError instanceof Error ? persistError.message : String(persistError);
+        logger.error(
+          `Failed to persist failed status for migration ${definition.id}: ${persistMessage}`
+        );
+      }
+      return {
+        success: false,
+        applied: false,
+        skipped: false,
+        name: definition.id,
+        checksum,
+        execution_time_ms: Date.now() - startTime,
+        error: migrationError
+      };
+    }
+  }
+  /**
+   * Apply multiple migrations in order
+   */
+  async applyAll(definitions, options = {}) {
+    if (options.atomic && !options.dryRun) {
+      if (!this.db.transaction) {
+        throw new Error(
+          "Atomic migration batches require a database adapter with transaction support."
+        );
+      }
+      if (this.dbEngine === "postgres") {
+        const concurrentStatement = findConcurrentIndexStatement(definitions);
+        if (concurrentStatement) {
+          throw new Error(
+            `Atomic migration batch cannot include CONCURRENTLY index DDL in ${concurrentStatement.migrationName}; PostgreSQL forbids CONCURRENTLY inside a transaction.`
+          );
+        }
+      }
+      try {
+        return await this.db.transaction(async (tx) => {
+          const txDb = {
+            ...tx,
+            transaction: async (callback) => callback(tx)
+          };
+          const txTracker = new MigrationTracker({
+            db: txDb,
+            lockTimeout: this.options.lockTimeout,
+            statementTimeout: this.options.statementTimeout,
+            useConcurrentIndexes: false
+          });
+          const results2 = await txTracker.applyAll(definitions, {
+            ...options,
+            atomic: false,
+            continueOnError: false,
+            postgresSafe: false
+          });
+          const failed = results2.find((result) => !result.success);
+          if (failed) {
+            throw new AtomicMigrationRollback(results2, failed);
+          }
+          return results2;
+        });
+      } catch (error) {
+        if (error instanceof AtomicMigrationRollback) {
+          return error.results.map((result) => {
+            if (!result.success || !result.applied) {
+              return result;
+            }
+            return {
+              ...result,
+              success: false,
+              applied: false,
+              skipped: false,
+              rolled_back: true,
+              error: new Error(
+                `Rolled back because migration ${error.failed.name} failed`
+              )
+            };
+          });
+        }
+        throw error;
+      }
+    }
+    const results = [];
+    this.currentBatch = await this.getNextBatch();
+    for (const definition of definitions) {
+      const result = await this.apply(definition, options);
+      results.push(result);
+      options.onProgress?.(result);
+      if (!result.success && !options.continueOnError) {
+        break;
+      }
+    }
+    this.currentBatch = null;
+    return results;
+  }
+  /**
+   * Rollback a migration
+   */
+  async rollback(name, definition, options = {}) {
+    await this.initialize();
+    const startTime = Date.now();
+    const checksum = computeChecksum(definition.down);
+    const record = await this.getMigration(name);
+    if (!record) {
+      return {
+        success: false,
+        name,
+        checksum,
+        execution_time_ms: 0,
+        error: new Error(`Migration ${name} not found in database`)
+      };
+    }
+    if (record.status !== "completed") {
+      return {
+        success: false,
+        name,
+        checksum,
+        execution_time_ms: 0,
+        error: new Error(
+          `Migration ${name} is not in completed state (current: ${record.status})`
+        )
+      };
+    }
+    if (!definition.down.length) {
+      return {
+        success: false,
+        name,
+        checksum,
+        execution_time_ms: 0,
+        error: new Error(
+          `Migration ${name} is not reversible (no down statements)`
+        )
+      };
+    }
+    if (options.dryRun) {
+      return {
+        success: true,
+        name,
+        checksum,
+        execution_time_ms: Date.now() - startTime,
+        rolled_back: true
+      };
+    }
+    try {
+      await this.executeStatements(definition.down, {});
+      await this.db.query(
+        `UPDATE _smrt_schema_migrations
+         SET status = 'rolled_back', rolled_back_at = CURRENT_TIMESTAMP
+         WHERE name = ?`,
+        name
+      );
+      return {
+        success: true,
+        name,
+        checksum,
+        execution_time_ms: Date.now() - startTime,
+        rolled_back: true
+      };
+    } catch (error) {
+      return {
+        success: false,
+        name,
+        checksum,
+        execution_time_ms: Date.now() - startTime,
+        error: error instanceof Error ? error : new Error(String(error)),
+        rolled_back: false
+      };
+    }
+  }
+  /**
+   * Get migration history with optional filtering
+   */
+  async getHistory(options = {}) {
+    await this.initialize();
+    let sql = `SELECT * FROM _smrt_schema_migrations WHERE 1=1`;
+    const params = [];
+    if (options.status) {
+      sql += ` AND status = ?`;
+      params.push(options.status);
+    }
+    if (options.since) {
+      sql += ` AND applied_at >= ?`;
+      params.push(options.since.toISOString());
+    }
+    sql += ` ORDER BY applied_at DESC`;
+    if (options.limit) {
+      sql += ` LIMIT ?`;
+      params.push(options.limit);
+    }
+    const result = await this.db.query(sql, ...params);
+    return result.rows.map((row) => ({
+      ...row,
+      applied_at: new Date(row.applied_at),
+      rolled_back_at: row.rolled_back_at ? new Date(row.rolled_back_at) : null,
+      is_reversible: Boolean(row.is_reversible)
+    }));
+  }
+  /**
+   * Execute SQL statements with appropriate transaction handling
+   */
+  async executeStatements(statements, options) {
+    if (this.dbEngine === "postgres" && options.postgresSafe) {
+      await this.executePostgresStatements(statements);
+    } else {
+      await this.executeWithTransaction(statements);
+    }
+  }
+  /**
+   * Execute statements within a transaction (SQLite/DuckDB)
+   */
+  async executeWithTransaction(statements) {
+    if (this.db.transaction) {
+      await this.db.transaction(async (tx) => {
+        for (const sql of statements) {
+          await tx.query(sql);
+        }
+      });
+    } else {
+      for (const sql of statements) {
+        await this.db.query(sql);
+      }
+    }
+  }
+  /**
+   * Execute statements with PostgreSQL-specific handling
+   *
+   * - CONCURRENTLY statements run outside transaction
+   * - Regular statements run in transaction with lock_timeout
+   *
+   * Both CREATE and DROP variants of CONCURRENTLY must be detected — Postgres
+   * forbids `DROP INDEX CONCURRENTLY` inside a transaction block, just like
+   * the create variant. Issue #1165: the migration generator emits
+   * `DROP INDEX CONCURRENTLY` for shape-drift recreates and orphan-index
+   * cleanups, so the regex needs to cover both keywords.
+   */
+  async executePostgresStatements(statements) {
+    const { concurrent: finalConcurrent, regular: finalRegular } = planPostgresStatements(statements, !!this.options.useConcurrentIndexes);
+    if (finalRegular.length > 0 && this.db.transaction) {
+      await this.db.transaction(async (tx) => {
+        await tx.query(
+          `SET LOCAL lock_timeout = '${this.options.lockTimeout}ms'`
+        );
+        await tx.query(
+          `SET LOCAL statement_timeout = '${this.options.statementTimeout}ms'`
+        );
+        for (const sql of finalRegular) {
+          await tx.query(sql);
+        }
+      });
+    } else if (finalRegular.length > 0) {
+      for (const sql of finalRegular) {
+        await this.db.query(sql);
+      }
+    }
+    for (const sql of finalConcurrent) {
+      await this.db.query(`SET lock_timeout = '${this.options.lockTimeout}ms'`);
+      await this.db.query(sql);
+    }
+  }
+}
+function planPostgresStatements(statements, useConcurrentIndexes) {
+  const concurrentRegex = CONCURRENT_INDEX_STATEMENT_RE;
+  const concurrent = [];
+  const regular = [];
+  for (const sql of statements) {
+    if (concurrentRegex.test(sql)) {
+      concurrent.push(sql);
+      continue;
+    }
+    if (useConcurrentIndexes) {
+      if (/CREATE\s+(UNIQUE\s+)?INDEX\s+(?!CONCURRENTLY)/i.test(sql)) {
+        concurrent.push(
+          sql.replace(
+            /CREATE\s+(UNIQUE\s+)?INDEX\s+/i,
+            "CREATE $1INDEX CONCURRENTLY "
+          )
+        );
+        continue;
+      }
+      if (/DROP\s+INDEX\s+(?!CONCURRENTLY)/i.test(sql)) {
+        concurrent.push(
+          sql.replace(/DROP\s+INDEX\s+/i, "DROP INDEX CONCURRENTLY ")
+        );
+        continue;
+      }
+    }
+    regular.push(sql);
+  }
+  return { concurrent, regular };
+}
+export {
+  MigrationTracker,
+  planPostgresStatements
+};
+//# sourceMappingURL=tracker.js.map