@prairielearn/migrations 1.0.0 → 1.2.0

package/src/{index.test.ts → load-migrations.test.ts}

@@ -4,11 +4,7 @@ import path from 'path';
 import tmp from 'tmp-promise';
 import fs from 'fs-extra';
 
-import {
-  readAndValidateMigrationsFromDirectory,
-  sortMigrationFiles,
-  getMigrationsToExecute,
-} from './index';
+import { readAndValidateMigrationsFromDirectory, sortMigrationFiles } from './load-migrations';
 
 chai.use(chaiAsPromised);
 
@@ -24,12 +20,12 @@ async function withMigrationFiles(files: string[], fn: (tmpDir: string) => Promi
   );
 }
 
-describe('migrations', () => {
+describe('load-migrations', () => {
   describe('readAndValidateMigrationsFromDirectory', () => {
     it('handles migrations without a timestamp', async () => {
       await withMigrationFiles(['001_testing.sql'], async (tmpDir) => {
         await assert.isRejected(
-          readAndValidateMigrationsFromDirectory(tmpDir),
+          readAndValidateMigrationsFromDirectory(tmpDir, ['.sql']),
           'Invalid migration filename: 001_testing.sql'
         );
       });
@@ -40,7 +36,7 @@ describe('migrations', () => {
         ['20220101010101_testing.sql', '20220101010101_testing_again.sql'],
         async (tmpDir) => {
           await assert.isRejected(
-            readAndValidateMigrationsFromDirectory(tmpDir),
+            readAndValidateMigrationsFromDirectory(tmpDir, ['.sql']),
             'Duplicate migration timestamp'
           );
         }
@@ -53,28 +49,34 @@ describe('migrations', () => {
       assert.deepEqual(
         sortMigrationFiles([
           {
+            directory: 'migrations',
             filename: '20220101010103_testing_3.sql',
             timestamp: '20220101010103',
           },
           {
+            directory: 'migrations',
             filename: '20220101010101_testing_1.sql',
             timestamp: '20220101010101',
           },
           {
+            directory: 'migrations',
             filename: '20220101010102_testing_2.sql',
             timestamp: '20220101010102',
           },
         ]),
         [
           {
+            directory: 'migrations',
             filename: '20220101010101_testing_1.sql',
             timestamp: '20220101010101',
           },
           {
+            directory: 'migrations',
             filename: '20220101010102_testing_2.sql',
             timestamp: '20220101010102',
           },
           {
+            directory: 'migrations',
             filename: '20220101010103_testing_3.sql',
             timestamp: '20220101010103',
           },
@@ -82,44 +84,4 @@ describe('migrations', () => {
       );
     });
   });
-
-  describe('getMigrationsToExecute', () => {
-    it('handles the case of no executed migrations', () => {
-      const migrationFiles = [
-        {
-          filename: '001_testing.sql',
-          timestamp: '20220101010101',
-        },
-      ];
-      assert.deepEqual(getMigrationsToExecute(migrationFiles, []), migrationFiles);
-    });
-
-    it('handles case where subset of migrations have been executed', () => {
-      const migrationFiles = [
-        {
-          filename: '20220101010101_testing_1.sql',
-          timestamp: '20220101010101',
-        },
-        {
-          filename: '20220101010102_testing_2.sql',
-          timestamp: '20220101010102',
-        },
-        {
-          filename: '20220101010103_testing_3.sql',
-          timestamp: '20220101010103',
-        },
-      ];
-      const executedMigrations = [
-        {
-          timestamp: '20220101010101',
-        },
-        {
-          timestamp: '20220101010102',
-        },
-      ];
-      assert.deepEqual(getMigrationsToExecute(migrationFiles, executedMigrations), [
-        { timestamp: '20220101010103', filename: '20220101010103_testing_3.sql' },
-      ]);
-    });
-  });
 });

package/src/load-migrations.ts

@@ -0,0 +1,76 @@
+import fs from 'fs-extra';
+
+/**
+ * Timestamp prefixes will be of the form `YYYYMMDDHHMMSS`, which will have 14 digits.
+ * If this code is still around in the year 10000... good luck.
+ */
+const MIGRATION_FILENAME_REGEX = /^([0-9]{14})_.+\.[a-z]+/;
+
+export interface MigrationFile {
+  directory: string;
+  filename: string;
+  timestamp: string;
+}
+
+export async function readAndValidateMigrationsFromDirectory(
+  dir: string,
+  extensions: string[]
+): Promise<MigrationFile[]> {
+  const migrationFiles = (await fs.readdir(dir)).filter((m) =>
+    extensions.some((e) => m.endsWith(e))
+  );
+
+  const migrations = migrationFiles.map((mf) => {
+    const match = mf.match(MIGRATION_FILENAME_REGEX);
+
+    if (!match) {
+      throw new Error(`Invalid migration filename: ${mf}`);
+    }
+
+    const timestamp = match[1] ?? null;
+
+    if (timestamp === null) {
+      throw new Error(`Migration ${mf} does not have a timestamp`);
+    }
+
+    return {
+      directory: dir,
+      filename: mf,
+      timestamp,
+    };
+  });
+
+  // First pass: validate that all migrations have a unique timestamp prefix.
+  // This will avoid data loss and conflicts in unexpected scenarios.
+  const seenTimestamps = new Set();
+  for (const migration of migrations) {
+    const { filename, timestamp } = migration;
+
+    if (timestamp !== null) {
+      if (seenTimestamps.has(timestamp)) {
+        throw new Error(`Duplicate migration timestamp: ${timestamp} (${filename})`);
+      }
+      seenTimestamps.add(timestamp);
+    }
+  }
+
+  return migrations;
+}
+
+export async function readAndValidateMigrationsFromDirectories(
+  directories: string[],
+  extensions: string[]
+): Promise<MigrationFile[]> {
+  const allMigrations: MigrationFile[] = [];
+  for (const directory of directories) {
+    const migrations = await readAndValidateMigrationsFromDirectory(directory, extensions);
+    allMigrations.push(...migrations);
+  }
+  return allMigrations;
+}
+
+export function sortMigrationFiles(migrationFiles: MigrationFile[]): MigrationFile[] {
+  return migrationFiles.sort((a, b) => {
+    return a.timestamp.localeCompare(b.timestamp);
+  });
+}

package/src/migrations/fixtures/20230407210409_create_users.sql

@@ -0,0 +1,2 @@
+CREATE TABLE IF NOT EXISTS
+  users (id BIGSERIAL PRIMARY KEY, name TEXT NOT NULL);

package/src/migrations/fixtures/20230407210430_insert_user.ts

@@ -0,0 +1,5 @@
+import { queryAsync } from '@prairielearn/postgres';
+
+module.exports = async function migrate() {
+  await queryAsync("INSERT INTO users (name) VALUES ('Test User')", {});
+};

package/src/migrations/index.ts

@@ -0,0 +1 @@
+export { init } from './migrations';

package/src/migrations/migrations.test.ts

@@ -0,0 +1,80 @@
+import { assert } from 'chai';
+import path from 'node:path';
+import { makePostgresTestUtils, queryAsync } from '@prairielearn/postgres';
+
+import { getMigrationsToExecute, initWithLock } from './migrations';
+
+describe('migrations', () => {
+  describe('getMigrationsToExecute', () => {
+    it('handles the case of no executed migrations', () => {
+      const migrationFiles = [
+        {
+          directory: 'migrations',
+          filename: '001_testing.sql',
+          timestamp: '20220101010101',
+        },
+      ];
+      assert.deepEqual(getMigrationsToExecute(migrationFiles, []), migrationFiles);
+    });
+
+    it('handles case where subset of migrations have been executed', () => {
+      const migrationFiles = [
+        {
+          directory: 'migrations',
+          filename: '20220101010101_testing_1.sql',
+          timestamp: '20220101010101',
+        },
+        {
+          directory: 'migrations',
+          filename: '20220101010102_testing_2.sql',
+          timestamp: '20220101010102',
+        },
+        {
+          directory: 'migrations',
+          filename: '20220101010103_testing_3.sql',
+          timestamp: '20220101010103',
+        },
+      ];
+      const executedMigrations = [
+        {
+          timestamp: '20220101010101',
+        },
+        {
+          timestamp: '20220101010102',
+        },
+      ];
+      assert.deepEqual(getMigrationsToExecute(migrationFiles, executedMigrations), [
+        {
+          directory: 'migrations',
+          timestamp: '20220101010103',
+          filename: '20220101010103_testing_3.sql',
+        },
+      ]);
+    });
+  });
+
+  describe('initWithLock', () => {
+    const postgresTestUtils = makePostgresTestUtils({
+      database: 'prairielearn_migrations',
+    });
+
+    before(async () => {
+      await postgresTestUtils.createDatabase();
+    });
+
+    after(async () => {
+      await postgresTestUtils.dropDatabase();
+    });
+
+    it('runs both SQL and JavaScript migrations', async () => {
+      const migrationDir = path.join(__dirname, 'fixtures');
+      await initWithLock([migrationDir], 'prairielearn_migrations');
+
+      // If both migrations ran successfully, there should be a single user
+      // in the database.
+      const users = await queryAsync('SELECT * FROM users', {});
+      assert.lengthOf(users.rows, 1);
+      assert.equal(users.rows[0].name, 'Test User');
+    });
+  });
+});

package/src/migrations/migrations.ts

@@ -0,0 +1,149 @@
+import fs from 'fs-extra';
+import path from 'path';
+
+import * as namedLocks from '@prairielearn/named-locks';
+import { logger } from '@prairielearn/logger';
+import * as sqldb from '@prairielearn/postgres';
+import * as error from '@prairielearn/error';
+
+import {
+  MigrationFile,
+  readAndValidateMigrationsFromDirectories,
+  sortMigrationFiles,
+} from '../load-migrations';
+
+const sql = sqldb.loadSqlEquiv(__filename);
+
+export async function init(directories: string | string[], project: string) {
+  const migrationDirectories = Array.isArray(directories) ? directories : [directories];
+  const lockName = 'migrations';
+  logger.verbose(`Waiting for lock ${lockName}`);
+  await namedLocks.doWithLock(
+    lockName,
+    {
+      // Migrations *might* take a long time to run, so we'll enable automatic
+      // lock renewal so that our lock doesn't get killed by the Postgres
+      // idle session timeout.
+      //
+      // That said, we should generally try to keep migrations executing as
+      // quickly as possible. A long-running migration likely means that
+      // Postgres is locking a whole table, which is unacceptable in production.
+      autoRenew: true,
+    },
+    async () => {
+      logger.verbose(`Acquired lock ${lockName}`);
+      await initWithLock(migrationDirectories, project);
+    }
+  );
+  logger.verbose(`Released lock ${lockName}`);
+}
+
+export function getMigrationsToExecute(
+  migrationFiles: MigrationFile[],
+  executedMigrations: { timestamp: string | null }[]
+): MigrationFile[] {
+  // If no migrations have ever been run, run them all.
+  if (executedMigrations.length === 0) {
+    return migrationFiles;
+  }
+
+  const executedMigrationTimestamps = new Set(executedMigrations.map((m) => m.timestamp));
+  return migrationFiles.filter((m) => !executedMigrationTimestamps.has(m.timestamp));
+}
+
+export async function initWithLock(directories: string[], project: string) {
+  logger.verbose('Starting DB schema migration');
+
+  // Create the migrations table if needed
+  await sqldb.queryAsync(sql.create_migrations_table, {});
+
+  // Apply necessary changes to the migrations table as needed.
+  try {
+    await sqldb.queryAsync('SELECT project FROM migrations;', {});
+  } catch (err: any) {
+    if (err.routine === 'errorMissingColumn') {
+      logger.info('Altering migrations table');
+      await sqldb.queryAsync(sql.add_projects_column, {});
+    } else {
+      throw err;
+    }
+  }
+  try {
+    await sqldb.queryAsync('SELECT timestamp FROM migrations;', {});
+  } catch (err: any) {
+    if (err.routine === 'errorMissingColumn') {
+      logger.info('Altering migrations table again');
+      await sqldb.queryAsync(sql.add_timestamp_column, {});
+    } else {
+      throw err;
+    }
+  }
+
+  let allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });
+
+  const migrationFiles = await readAndValidateMigrationsFromDirectories(directories, [
+    '.sql',
+    '.js',
+    '.ts',
+    '.mjs',
+  ]);
+
+  // Validation: if we not all previously-executed migrations have timestamps,
+  // prompt the user to deploy an earlier version that includes both indexes
+  // and timestamps.
+  const migrationsMissingTimestamps = allMigrations.rows.filter((m) => !m.timestamp);
+  if (migrationsMissingTimestamps.length > 0) {
+    throw new Error(
+      [
+        'The following migrations are missing timestamps:',
+        migrationsMissingTimestamps.map((m) => `  ${m.filename}`),
+        // This revision was the most recent commit to `master` before the
+        // code handling indexes was removed.
+        'You must deploy revision 1aa43c7348fa24cf636413d720d06a2fa9e38ef2 first.',
+      ].join('\n')
+    );
+  }
+
+  // Refetch the list of migrations from the database.
+  allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });
+
+  // Sort the migration files into execution order.
+  const sortedMigrationFiles = sortMigrationFiles(migrationFiles);
+
+  // Figure out which migrations have to be applied.
+  const migrationsToExecute = getMigrationsToExecute(sortedMigrationFiles, allMigrations.rows);
+
+  for (const { directory, filename, timestamp } of migrationsToExecute) {
+    if (allMigrations.rows.length === 0) {
+      // if we are running all the migrations then log at a lower level
+      logger.verbose(`Running migration ${filename}`);
+    } else {
+      logger.info(`Running migration ${filename}`);
+    }
+
+    const migrationPath = path.join(directory, filename);
+    if (filename.endsWith('.sql')) {
+      const migrationSql = await fs.readFile(migrationPath, 'utf8');
+      try {
+        await sqldb.queryAsync(migrationSql, {});
+      } catch (err) {
+        error.addData(err, { sqlFile: filename });
+        throw err;
+      }
+    } else {
+      const migrationModule = await import(migrationPath);
+      const implementation = migrationModule.default;
+      if (typeof implementation !== 'function') {
+        throw new Error(`Migration ${filename} does not export a default function`);
+      }
+      await implementation();
+    }
+
+    // Record the migration.
+    await sqldb.queryAsync(sql.insert_migration, {
+      filename: filename,
+      timestamp,
+      project,
+    });
+  }
+}

package/dist/index.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAoC;AACpC,wEAA8C;AAC9C,gDAAwB;AACxB,8DAA8B;AAC9B,wDAA0B;AAE1B,mCAIiB;AAEjB,cAAI,CAAC,GAAG,CAAC,0BAAc,CAAC,CAAC;AAEzB,KAAK,UAAU,kBAAkB,CAAC,KAAe,EAAE,EAAqC;IACtF,MAAM,qBAAG,CAAC,OAAO,CACf,KAAK,WAAW,MAAM;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,MAAM,kBAAE,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;SACtD;QACD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,CACxB,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,wCAAwC,EAAE,GAAG,EAAE;QACtD,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,kBAAkB,CAAC,CAAC,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;gBAC7D,MAAM,aAAM,CAAC,UAAU,CACrB,IAAA,8CAAsC,EAAC,MAAM,CAAC,EAC9C,6CAA6C,CAC9C,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,kBAAkB,CACtB,CAAC,4BAA4B,EAAE,kCAAkC,CAAC,EAClE,KAAK,EAAE,MAAM,EAAE,EAAE;gBACf,MAAM,aAAM,CAAC,UAAU,CACrB,IAAA,8CAAsC,EAAC,MAAM,CAAC,EAC9C,+BAA+B,CAChC,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,aAAM,CAAC,SAAS,CACd,IAAA,0BAAkB,EAAC;gBACjB;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC,EACF;gBACE;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,cAAc,GAAG;gBACrB;oBACE,QAAQ,EAAE,iBAAiB;oBAC3B,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,8BAAsB,EAAC,cAAc,EAAE,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,cAAc,GAAG;gBACrB;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG;gBACzB;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,8BAAsB,EAAC,cAAc,EAAE,kBAAkB,CAAC,EAAE;gBAC3E,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,8BAA8B,EAAE;aAC1E,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}