@prairielearn/migrations 1.1.0 → 1.2.1

package/dist/migrations/migrations.test.js

@@ -0,0 +1,78 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const node_path_1 = __importDefault(require("node:path"));
+const postgres_1 = require("@prairielearn/postgres");
+const migrations_1 = require("./migrations");
+describe('migrations', () => {
+    describe('getMigrationsToExecute', () => {
+        it('handles the case of no executed migrations', () => {
+            const migrationFiles = [
+                {
+                    directory: 'migrations',
+                    filename: '001_testing.sql',
+                    timestamp: '20220101010101',
+                },
+            ];
+            chai_1.assert.deepEqual((0, migrations_1.getMigrationsToExecute)(migrationFiles, []), migrationFiles);
+        });
+        it('handles case where subset of migrations have been executed', () => {
+            const migrationFiles = [
+                {
+                    directory: 'migrations',
+                    filename: '20220101010101_testing_1.sql',
+                    timestamp: '20220101010101',
+                },
+                {
+                    directory: 'migrations',
+                    filename: '20220101010102_testing_2.sql',
+                    timestamp: '20220101010102',
+                },
+                {
+                    directory: 'migrations',
+                    filename: '20220101010103_testing_3.sql',
+                    timestamp: '20220101010103',
+                },
+            ];
+            const executedMigrations = [
+                {
+                    timestamp: '20220101010101',
+                },
+                {
+                    timestamp: '20220101010102',
+                },
+            ];
+            chai_1.assert.deepEqual((0, migrations_1.getMigrationsToExecute)(migrationFiles, executedMigrations), [
+                {
+                    directory: 'migrations',
+                    timestamp: '20220101010103',
+                    filename: '20220101010103_testing_3.sql',
+                },
+            ]);
+        });
+    });
+    describe('initWithLock', () => {
+        const postgresTestUtils = (0, postgres_1.makePostgresTestUtils)({
+            database: 'prairielearn_migrations',
+        });
+        before(async () => {
+            await postgresTestUtils.createDatabase();
+        });
+        after(async () => {
+            await postgresTestUtils.dropDatabase();
+        });
+        it('runs both SQL and JavaScript migrations', async () => {
+            const migrationDir = node_path_1.default.join(__dirname, 'fixtures');
+            await (0, migrations_1.initWithLock)([migrationDir], 'prairielearn_migrations');
+            // If both migrations ran successfully, there should be a single user
+            // in the database.
+            const users = await (0, postgres_1.queryAsync)('SELECT * FROM users', {});
+            chai_1.assert.lengthOf(users.rows, 1);
+            chai_1.assert.equal(users.rows[0].name, 'Test User');
+        });
+    });
+});
+//# sourceMappingURL=migrations.test.js.map

package/dist/migrations/migrations.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"migrations.test.js","sourceRoot":"","sources":["../../src/migrations/migrations.test.ts"],"names":[],"mappings":";;;;;AAAA,+BAA8B;AAC9B,0DAA6B;AAC7B,qDAA2E;AAE3E,6CAAoE;AAEpE,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,cAAc,GAAG;gBACrB;oBACE,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,iBAAiB;oBAC3B,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,mCAAsB,EAAC,cAAc,EAAE,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,cAAc,GAAG;gBACrB;oBACE,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,SAAS,EAAE,YAAY;oBACvB,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG;gBACzB;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,mCAAsB,EAAC,cAAc,EAAE,kBAAkB,CAAC,EAAE;gBAC3E;oBACE,SAAS,EAAE,YAAY;oBACvB,SAAS,EAAE,gBAAgB;oBAC3B,QAAQ,EAAE,8BAA8B;iBACzC;aACF,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,MAAM,iBAAiB,GAAG,IAAA,gCAAqB,EAAC;YAC9C,QAAQ,EAAE,yBAAyB;SACpC,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,IAAI,EAAE;YAChB,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,GAAG,mBAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACtD,MAAM,IAAA,yBAAY,EAAC,CAAC,YAAY,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAE9D,qEAAqE;YACrE,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,IAAA,qBAAU,EAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;YAC1D,aAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/B,aAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prairielearn/migrations",
-  "version": "1.1.0",
+  "version": "1.2.1",
   "main": "./dist/index.js",
   "repository": {
     "type": "git",
@@ -8,24 +8,27 @@
     "directory": "packages/migrations"
   },
   "scripts": {
-    "build": "tsc && copyfiles -u 1 \"./src/**/*.sql\" dist",
-    "dev": "tsc --watch --preserveWatchOutput",
-    "test": "mocha --no-config --require ts-node/register src/*.test.ts"
+    "build": "tsc && tscp",
+    "dev": "tsc --watch --preserveWatchOutput & tscp --watch",
+    "test": "mocha --no-config --require ts-node/register src/**/*.test.ts"
   },
   "devDependencies": {
-    "@prairielearn/tsconfig": "*",
+    "@prairielearn/tsconfig": "^0.0.0",
     "@types/fs-extra": "^11.0.1",
     "@types/mocha": "^10.0.1",
-    "@types/node": "^18.15.11",
-    "copyfiles": "^2.4.1",
+    "@types/node": "^18.16.3",
     "mocha": "^10.2.0",
     "ts-node": "^10.9.1",
-    "typescript": "^4.9.5"
+    "typescript": "^5.0.4",
+    "typescript-cp": "^0.1.7"
   },
   "dependencies": {
-    "@prairielearn/error": "^1.0.0",
-    "@prairielearn/named-locks": "^1.2.0",
-    "@prairielearn/postgres": "^1.2.0",
-    "fs-extra": "^11.1.1"
+    "@prairielearn/error": "^1.0.1",
+    "@prairielearn/logger": "^1.0.0",
+    "@prairielearn/named-locks": "^1.3.1",
+    "@prairielearn/postgres": "^1.6.1",
+    "fs-extra": "^11.1.1",
+    "serialize-error": "^8.1.0",
+    "zod": "^3.21.4"
   }
 }

package/schema-migrations/20230303193423_batched_migrations__create.sql

@@ -0,0 +1,49 @@
+CREATE TYPE enum_batched_migration_status AS ENUM(
+  'pending',
+  'paused',
+  'running',
+  'finalizing',
+  'failed',
+  'succeeded'
+);
+
+CREATE TYPE enum_batched_migration_job_status AS ENUM('pending', 'failed', 'succeeded');
+
+CREATE TABLE IF NOT EXISTS
+  batched_migrations (
+    id BIGSERIAL PRIMARY KEY,
+    project TEXT DEFAULT 'prairielearn' NOT NULL,
+    filename TEXT NOT NULL,
+    timestamp TEXT NOT NULL,
+    batch_size INTEGER NOT NULL,
+    min_value BIGINT NOT NULL,
+    max_value BIGINT NOT NULL,
+    status enum_batched_migration_status DEFAULT 'pending' NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    started_at TIMESTAMP WITH TIME ZONE,
+    UNIQUE (project, timestamp),
+    CONSTRAINT batched_migrations_min_value_check CHECK (min_value > 0),
+    CONSTRAINT batched_migrations_max_value_check CHECK (max_value >= min_value)
+  );
+
+CREATE TABLE IF NOT EXISTS
+  batched_migration_jobs (
+    id BIGSERIAL PRIMARY KEY,
+    batched_migration_id BIGINT NOT NULL REFERENCES batched_migrations (id) ON UPDATE CASCADE ON DELETE CASCADE,
+    min_value BIGINT NOT NULL,
+    max_value BIGINT NOT NULL,
+    status enum_batched_migration_job_status DEFAULT 'pending' NOT NULL,
+    attempts INTEGER DEFAULT 0 NOT NULL,
+    created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    started_at TIMESTAMP WITH TIME ZONE,
+    finished_at TIMESTAMP WITH TIME ZONE,
+    data jsonb,
+    CONSTRAINT batched_migration_jobs_min_value_check CHECK (min_value > 0),
+    CONSTRAINT batched_migration_jobs_max_value_check CHECK (max_value >= min_value)
+  );
+
+CREATE INDEX IF NOT EXISTS batched_migration_jobs_batched_migration_id_max_value_idx ON batched_migration_jobs (batched_migration_id, max_value);
+
+CREATE INDEX IF NOT EXISTS batched_migration_jobs_batched_migration_id_status_idx ON batched_migration_jobs (batched_migration_id, status);

package/src/batched-migrations/batched-migration-job.sql

@@ -0,0 +1,12 @@
+-- BLOCK select_recent_jobs_with_status
+SELECT
+  *
+FROM
+  batched_migration_jobs
+WHERE
+  batched_migration_id = $batched_migration_id
+  AND status = $status
+ORDER BY
+  max_value DESC
+LIMIT
+  $limit;

package/src/batched-migrations/batched-migration-job.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+import { loadSqlEquiv, queryValidatedRows } from '@prairielearn/postgres';
+
+const sql = loadSqlEquiv(__filename);
+
+export const BatchedMigrationJobStatusSchema = z.enum(['pending', 'failed', 'succeeded']);
+export type BatchedMigrationJobStatus = z.infer<typeof BatchedMigrationJobStatusSchema>;
+
+export const BatchedMigrationJobRowSchema = z.object({
+  id: z.string(),
+  batched_migration_id: z.string(),
+  min_value: z.bigint({ coerce: true }),
+  max_value: z.bigint({ coerce: true }),
+  status: BatchedMigrationJobStatusSchema,
+  attempts: z.number(),
+  created_at: z.date(),
+  updated_at: z.date(),
+  started_at: z.date().nullable(),
+  finished_at: z.date().nullable(),
+  data: z.unknown(),
+});
+export type BatchedMigrationJobRow = z.infer<typeof BatchedMigrationJobRowSchema>;
+
+export async function selectRecentJobsWithStatus(
+  batchedMigrationId: string,
+  status: BatchedMigrationJobStatus,
+  limit: number
+): Promise<BatchedMigrationJobRow[]> {
+  return queryValidatedRows(
+    sql.select_recent_jobs_with_status,
+    { batched_migration_id: batchedMigrationId, status, limit },
+    BatchedMigrationJobRowSchema
+  );
+}

package/src/batched-migrations/batched-migration-runner.sql

@@ -0,0 +1,93 @@
+-- BLOCK select_last_batched_migration_job
+SELECT
+  *
+FROM
+  batched_migration_jobs
+WHERE
+  batched_migration_id = $batched_migration_id
+ORDER BY
+  id DESC
+LIMIT
+  1;
+
+-- BLOCK insert_batched_migration_job
+INSERT INTO
+  batched_migration_jobs (
+    batched_migration_id,
+    status,
+    min_value,
+    max_value
+  )
+VALUES
+  (
+    $batched_migration_id,
+    'pending'::enum_batched_migration_job_status,
+    $min_value,
+    $max_value
+  )
+RETURNING
+  *;
+
+-- BLOCK start_batched_migration_job
+UPDATE batched_migration_jobs
+SET
+  attempts = attempts + 1,
+  updated_at = CURRENT_TIMESTAMP,
+  started_at = CURRENT_TIMESTAMP
+WHERE
+  id = $id;
+
+-- BLOCK finish_batched_migration_job
+UPDATE batched_migration_jobs
+SET
+  status = $status::enum_batched_migration_job_status,
+  updated_at = CURRENT_TIMESTAMP,
+  finished_at = CURRENT_TIMESTAMP,
+  data = $data
+WHERE
+  id = $id;
+
+-- BLOCK select_first_pending_batched_migration_job
+SELECT
+  *
+FROM
+  batched_migration_jobs
+WHERE
+  batched_migration_id = $batched_migration_id
+  AND status = 'pending'
+ORDER BY
+  id ASC
+LIMIT
+  1;
+
+-- BLOCK batched_migration_has_incomplete_jobs
+SELECT
+  EXISTS (
+    SELECT
+      1
+    FROM
+      batched_migration_jobs
+    WHERE
+      batched_migration_id = $batched_migration_id
+      AND status = 'pending'
+  ) as exists;
+
+-- BLOCK batched_migration_has_failed_jobs
+SELECT
+  EXISTS (
+    SELECT
+      1
+    FROM
+      batched_migration_jobs
+    WHERE
+      batched_migration_id = $batched_migration_id
+      AND status = 'failed'
+  ) as exists;
+
+-- BLOCK get_migration_status
+SELECT
+  status
+FROM
+  batched_migrations
+WHERE
+  id = $id;

package/src/batched-migrations/batched-migration-runner.test.ts

@@ -0,0 +1,208 @@
+import { assert } from 'chai';
+import {
+  makePostgresTestUtils,
+  queryAsync,
+  queryValidatedOneRow,
+  queryValidatedRows,
+} from '@prairielearn/postgres';
+import * as namedLocks from '@prairielearn/named-locks';
+import * as error from '@prairielearn/error';
+
+import {
+  BatchedMigrationRowSchema,
+  insertBatchedMigration,
+  makeBatchedMigration,
+  updateBatchedMigrationStatus,
+} from './batched-migration';
+import { BatchedMigrationJobRowSchema } from './batched-migration-job';
+import { BatchedMigrationRunner } from './batched-migration-runner';
+import { SCHEMA_MIGRATIONS_PATH, init } from '../index';
+
+const postgresTestUtils = makePostgresTestUtils({
+  database: 'prairielearn_migrations',
+});
+
+function makeTestBatchMigration() {
+  let executionCount = 0;
+  let failingIds: bigint[] = [];
+
+  return makeBatchedMigration({
+    async getParameters() {
+      return {
+        min: 1n,
+        max: 10000n,
+        batchSize: 1000,
+      };
+    },
+    async execute(start: bigint, end: bigint) {
+      executionCount += 1;
+      const shouldFail = failingIds.some((id) => id >= start && id <= end);
+      if (shouldFail) {
+        // Throw an error with some data to make sure it gets persisted. We
+        // specifically use BigInt values here to make sure that they are
+        // correctly serialized to strings.
+        throw error.makeWithData('Execution failure', { start, end });
+      }
+    },
+    setFailingIds(ids: bigint[]) {
+      failingIds = ids;
+    },
+    get executionCount() {
+      return executionCount;
+    },
+  });
+}
+
+async function getBatchedMigration(migrationId: string) {
+  return queryValidatedOneRow(
+    'SELECT * FROM batched_migrations WHERE id = $id;',
+    { id: migrationId },
+    BatchedMigrationRowSchema
+  );
+}
+
+async function getBatchedMigrationJobs(migrationId: string) {
+  return queryValidatedRows(
+    'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',
+    { batched_migration_id: migrationId },
+    BatchedMigrationJobRowSchema
+  );
+}
+
+async function resetFailedBatchedMigrationJobs(migrationId: string) {
+  await queryAsync(
+    "UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'",
+    {
+      batched_migration_id: migrationId,
+    }
+  );
+}
+
+async function insertTestBatchedMigration() {
+  const migrationImplementation = makeTestBatchMigration();
+  const parameters = await migrationImplementation.getParameters();
+  const migration = await insertBatchedMigration({
+    project: 'test',
+    filename: '20230406184103_test_batch_migration.js',
+    timestamp: '20230406184103',
+    batch_size: parameters.batchSize,
+    min_value: parameters.min,
+    max_value: parameters.max,
+    status: 'running',
+  });
+  if (!migration) throw new Error('Failed to insert batched migration');
+  return migration;
+}
+
+describe('BatchedMigrationExecutor', () => {
+  before(async () => {
+    await postgresTestUtils.createDatabase();
+    await namedLocks.init(postgresTestUtils.getPoolConfig(), (err) => {
+      throw err;
+    });
+    await init([SCHEMA_MIGRATIONS_PATH], 'prairielearn_migrations');
+  });
+
+  beforeEach(async () => {
+    await postgresTestUtils.resetDatabase();
+  });
+
+  after(async () => {
+    await namedLocks.close();
+    await postgresTestUtils.dropDatabase();
+  });
+
+  it('runs one iteration of a batched migration', async () => {
+    const migration = await insertTestBatchedMigration();
+
+    const migrationImplementation = makeTestBatchMigration();
+    const executor = new BatchedMigrationRunner(migration, migrationImplementation);
+    await executor.run({ iterations: 1 });
+
+    const jobs = await getBatchedMigrationJobs(migration.id);
+    assert.lengthOf(jobs, 1);
+
+    const finalMigration = await getBatchedMigration(migration.id);
+    assert.equal(finalMigration.status, 'running');
+
+    assert.equal(migrationImplementation.executionCount, 1);
+  });
+
+  it('runs an entire batched migration', async () => {
+    const migration = await insertTestBatchedMigration();
+
+    const migrationImplementation = makeTestBatchMigration();
+    const runner = new BatchedMigrationRunner(migration, migrationImplementation);
+    await runner.run();
+
+    const jobs = await getBatchedMigrationJobs(migration.id);
+    assert.lengthOf(jobs, 10);
+    assert.equal(jobs[0].min_value, 1n);
+    assert.equal(jobs[0].max_value, 1000n);
+    assert.equal(jobs.at(-1)?.min_value, 9001n);
+    assert.equal(jobs.at(-1)?.max_value, 10000n);
+    assert.isTrue(jobs.every((job) => job.started_at !== null));
+    assert.isTrue(jobs.every((job) => job.finished_at !== null));
+    assert.isTrue(jobs.every((job) => job.status === 'succeeded'));
+    assert.isTrue(jobs.every((job) => job.attempts === 1));
+
+    const finalMigration = await getBatchedMigration(migration.id);
+    assert.equal(finalMigration.status, 'succeeded');
+  });
+
+  it('handles failing execution', async () => {
+    let migration = await insertTestBatchedMigration();
+
+    const migrationImplementation = makeTestBatchMigration();
+    migrationImplementation.setFailingIds([1n, 5010n]);
+    const runner = new BatchedMigrationRunner(migration, migrationImplementation);
+    await runner.run();
+
+    const jobs = await getBatchedMigrationJobs(migration.id);
+    const failedJobs = jobs.filter((job) => job.status === 'failed');
+    const successfulJobs = jobs.filter((job) => job.status === 'succeeded');
+    assert.lengthOf(jobs, 10);
+    assert.lengthOf(failedJobs, 2);
+    assert.lengthOf(successfulJobs, 8);
+    assert.equal(migrationImplementation.executionCount, 10);
+    assert.isTrue(jobs.every((job) => job.attempts === 1));
+    failedJobs.forEach((job) => {
+      const jobData = job.data as any;
+      assert.isObject(jobData);
+      assert.isObject(jobData.error);
+      assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data']);
+      assert.equal(jobData.error.name, 'Error');
+      assert.equal(jobData.error.message, 'Execution failure');
+      assert.equal(jobData.error.data.start, job.min_value.toString());
+      assert.equal(jobData.error.data.end, job.max_value.toString());
+    });
+
+    const failedMigration = await getBatchedMigration(migration.id);
+    assert.equal(failedMigration.status, 'failed');
+
+    // Retry the failed jobs; ensure they succeed this time.
+    await resetFailedBatchedMigrationJobs(migration.id);
+    migration = await updateBatchedMigrationStatus(migration.id, 'running');
+
+    migrationImplementation.setFailingIds([]);
+    const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);
+    await retryRunner.run();
+
+    const finalJobs = await getBatchedMigrationJobs(migration.id);
+    const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');
+    const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');
+    const retriedJobs = finalJobs.filter((job) => job.attempts === 2);
+    assert.lengthOf(finalJobs, 10);
+    assert.lengthOf(finalFailedJobs, 0);
+    assert.lengthOf(finalSuccessfulJobs, 10);
+    assert.lengthOf(retriedJobs, 2);
+    assert.isTrue(finalJobs.every((job) => job.data === null));
+
+    migration = await getBatchedMigration(migration.id);
+    assert.equal(migration.status, 'succeeded');
+
+    // The runner should have run only the previously failed jobs, which
+    // works out to 2 additional execution.
+    assert.equal(migrationImplementation.executionCount, 12);
+  });
+});