@prairielearn/migrations 1.1.0 → 1.2.1

package/dist/batched-migrations/batched-migration-runner.test.js

@@ -0,0 +1,185 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const postgres_1 = require("@prairielearn/postgres");
+const namedLocks = __importStar(require("@prairielearn/named-locks"));
+const error = __importStar(require("@prairielearn/error"));
+const batched_migration_1 = require("./batched-migration");
+const batched_migration_job_1 = require("./batched-migration-job");
+const batched_migration_runner_1 = require("./batched-migration-runner");
+const index_1 = require("../index");
+const postgresTestUtils = (0, postgres_1.makePostgresTestUtils)({
+    database: 'prairielearn_migrations',
+});
+function makeTestBatchMigration() {
+    let executionCount = 0;
+    let failingIds = [];
+    return (0, batched_migration_1.makeBatchedMigration)({
+        async getParameters() {
+            return {
+                min: 1n,
+                max: 10000n,
+                batchSize: 1000,
+            };
+        },
+        async execute(start, end) {
+            executionCount += 1;
+            const shouldFail = failingIds.some((id) => id >= start && id <= end);
+            if (shouldFail) {
+                // Throw an error with some data to make sure it gets persisted. We
+                // specifically use BigInt values here to make sure that they are
+                // correctly serialized to strings.
+                throw error.makeWithData('Execution failure', { start, end });
+            }
+        },
+        setFailingIds(ids) {
+            failingIds = ids;
+        },
+        get executionCount() {
+            return executionCount;
+        },
+    });
+}
+async function getBatchedMigration(migrationId) {
+    return (0, postgres_1.queryValidatedOneRow)('SELECT * FROM batched_migrations WHERE id = $id;', { id: migrationId }, batched_migration_1.BatchedMigrationRowSchema);
+}
+async function getBatchedMigrationJobs(migrationId) {
+    return (0, postgres_1.queryValidatedRows)('SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;', { batched_migration_id: migrationId }, batched_migration_job_1.BatchedMigrationJobRowSchema);
+}
+async function resetFailedBatchedMigrationJobs(migrationId) {
+    await (0, postgres_1.queryAsync)("UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'", {
+        batched_migration_id: migrationId,
+    });
+}
+async function insertTestBatchedMigration() {
+    const migrationImplementation = makeTestBatchMigration();
+    const parameters = await migrationImplementation.getParameters();
+    const migration = await (0, batched_migration_1.insertBatchedMigration)({
+        project: 'test',
+        filename: '20230406184103_test_batch_migration.js',
+        timestamp: '20230406184103',
+        batch_size: parameters.batchSize,
+        min_value: parameters.min,
+        max_value: parameters.max,
+        status: 'running',
+    });
+    if (!migration)
+        throw new Error('Failed to insert batched migration');
+    return migration;
+}
+describe('BatchedMigrationExecutor', () => {
+    before(async () => {
+        await postgresTestUtils.createDatabase();
+        await namedLocks.init(postgresTestUtils.getPoolConfig(), (err) => {
+            throw err;
+        });
+        await (0, index_1.init)([index_1.SCHEMA_MIGRATIONS_PATH], 'prairielearn_migrations');
+    });
+    beforeEach(async () => {
+        await postgresTestUtils.resetDatabase();
+    });
+    after(async () => {
+        await namedLocks.close();
+        await postgresTestUtils.dropDatabase();
+    });
+    it('runs one iteration of a batched migration', async () => {
+        const migration = await insertTestBatchedMigration();
+        const migrationImplementation = makeTestBatchMigration();
+        const executor = new batched_migration_runner_1.BatchedMigrationRunner(migration, migrationImplementation);
+        await executor.run({ iterations: 1 });
+        const jobs = await getBatchedMigrationJobs(migration.id);
+        chai_1.assert.lengthOf(jobs, 1);
+        const finalMigration = await getBatchedMigration(migration.id);
+        chai_1.assert.equal(finalMigration.status, 'running');
+        chai_1.assert.equal(migrationImplementation.executionCount, 1);
+    });
+    it('runs an entire batched migration', async () => {
+        const migration = await insertTestBatchedMigration();
+        const migrationImplementation = makeTestBatchMigration();
+        const runner = new batched_migration_runner_1.BatchedMigrationRunner(migration, migrationImplementation);
+        await runner.run();
+        const jobs = await getBatchedMigrationJobs(migration.id);
+        chai_1.assert.lengthOf(jobs, 10);
+        chai_1.assert.equal(jobs[0].min_value, 1n);
+        chai_1.assert.equal(jobs[0].max_value, 1000n);
+        chai_1.assert.equal(jobs.at(-1)?.min_value, 9001n);
+        chai_1.assert.equal(jobs.at(-1)?.max_value, 10000n);
+        chai_1.assert.isTrue(jobs.every((job) => job.started_at !== null));
+        chai_1.assert.isTrue(jobs.every((job) => job.finished_at !== null));
+        chai_1.assert.isTrue(jobs.every((job) => job.status === 'succeeded'));
+        chai_1.assert.isTrue(jobs.every((job) => job.attempts === 1));
+        const finalMigration = await getBatchedMigration(migration.id);
+        chai_1.assert.equal(finalMigration.status, 'succeeded');
+    });
+    it('handles failing execution', async () => {
+        let migration = await insertTestBatchedMigration();
+        const migrationImplementation = makeTestBatchMigration();
+        migrationImplementation.setFailingIds([1n, 5010n]);
+        const runner = new batched_migration_runner_1.BatchedMigrationRunner(migration, migrationImplementation);
+        await runner.run();
+        const jobs = await getBatchedMigrationJobs(migration.id);
+        const failedJobs = jobs.filter((job) => job.status === 'failed');
+        const successfulJobs = jobs.filter((job) => job.status === 'succeeded');
+        chai_1.assert.lengthOf(jobs, 10);
+        chai_1.assert.lengthOf(failedJobs, 2);
+        chai_1.assert.lengthOf(successfulJobs, 8);
+        chai_1.assert.equal(migrationImplementation.executionCount, 10);
+        chai_1.assert.isTrue(jobs.every((job) => job.attempts === 1));
+        failedJobs.forEach((job) => {
+            const jobData = job.data;
+            chai_1.assert.isObject(jobData);
+            chai_1.assert.isObject(jobData.error);
+            chai_1.assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data']);
+            chai_1.assert.equal(jobData.error.name, 'Error');
+            chai_1.assert.equal(jobData.error.message, 'Execution failure');
+            chai_1.assert.equal(jobData.error.data.start, job.min_value.toString());
+            chai_1.assert.equal(jobData.error.data.end, job.max_value.toString());
+        });
+        const failedMigration = await getBatchedMigration(migration.id);
+        chai_1.assert.equal(failedMigration.status, 'failed');
+        // Retry the failed jobs; ensure they succeed this time.
+        await resetFailedBatchedMigrationJobs(migration.id);
+        migration = await (0, batched_migration_1.updateBatchedMigrationStatus)(migration.id, 'running');
+        migrationImplementation.setFailingIds([]);
+        const retryRunner = new batched_migration_runner_1.BatchedMigrationRunner(migration, migrationImplementation);
+        await retryRunner.run();
+        const finalJobs = await getBatchedMigrationJobs(migration.id);
+        const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');
+        const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');
+        const retriedJobs = finalJobs.filter((job) => job.attempts === 2);
+        chai_1.assert.lengthOf(finalJobs, 10);
+        chai_1.assert.lengthOf(finalFailedJobs, 0);
+        chai_1.assert.lengthOf(finalSuccessfulJobs, 10);
+        chai_1.assert.lengthOf(retriedJobs, 2);
+        chai_1.assert.isTrue(finalJobs.every((job) => job.data === null));
+        migration = await getBatchedMigration(migration.id);
+        chai_1.assert.equal(migration.status, 'succeeded');
+        // The runner should have run only the previously failed jobs, which
+        // works out to 2 additional execution.
+        chai_1.assert.equal(migrationImplementation.executionCount, 12);
+    });
+});
+//# sourceMappingURL=batched-migration-runner.test.js.map

package/dist/batched-migrations/batched-migration-runner.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batched-migration-runner.test.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+BAA8B;AAC9B,qDAKgC;AAChC,sEAAwD;AACxD,2DAA6C;AAE7C,2DAK6B;AAC7B,mEAAuE;AACvE,yEAAoE;AACpE,oCAAwD;AAExD,MAAM,iBAAiB,GAAG,IAAA,gCAAqB,EAAC;IAC9C,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AAEH,SAAS,sBAAsB;IAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,OAAO,IAAA,wCAAoB,EAAC;QAC1B,KAAK,CAAC,aAAa;YACjB,OAAO;gBACL,GAAG,EAAE,EAAE;gBACP,GAAG,EAAE,MAAM;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,GAAW;YACtC,cAAc,IAAI,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;YACrE,IAAI,UAAU,EAAE;gBACd,mEAAmE;gBACnE,iEAAiE;gBACjE,mCAAmC;gBACnC,MAAM,KAAK,CAAC,YAAY,CAAC,mBAAmB,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;aAC/D;QACH,CAAC;QACD,aAAa,CAAC,GAAa;YACzB,UAAU,GAAG,GAAG,CAAC;QACnB,CAAC;QACD,IAAI,cAAc;YAChB,OAAO,cAAc,CAAC;QACxB,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB;IACpD,OAAO,IAAA,+BAAoB,EACzB,kDAAkD,EAClD,EAAE,EAAE,EAAE,WAAW,EAAE,EACnB,6CAAyB,CAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB;IACxD,OAAO,IAAA,6BAAkB,EACvB,0GAA0G,EAC1G,EAAE,oBAAoB,EAAE,WAAW,EAAE,EACrC,oDAA4B,CAC7B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,+BAA+B,CAAC,WAAmB;IAChE,MAAM,IAAA,qBAAU,EACd,+JAA+J,EAC/J;QACE,oBAAoB,EAAE,WAAW;KAClC,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,0BAA0B;IACvC,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,IAAA,0CAAsB,EAAC;QAC7C,OAAO,EAAE,MAAM;QACf,QAAQ,EAAE,wCAAwC;QAClD,SAAS,EAAE,gBAAgB;QAC3B,UAAU,EAAE,UAAU,CAAC,SAAS;QAChC,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,MAAM,CAAC,KAAK,IAAI,EAAE;QAChB,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QACzC,MAAM,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,MAAM,IAAA,YAAI,EAAC,CAAC,8BAAsB,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,iDAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAChF,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,aAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,aAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/C,aAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,iDAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,aAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,aAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,aAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,aAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,aAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,aAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;QAC5D,aAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC;QAC7D,aAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC;QAC/D,aAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,aAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,IAAI,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAEnD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,uBAAuB,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,iDAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACxE,aAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,aAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/B,aAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnC,aAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACzD,aAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,GAAG,CAAC,IAAW,CAAC;YAChC,aAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzB,aAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,aAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;YACvE,aAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,aAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;YACzD,aAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,aAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAChE,aAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE/C,wDAAwD;QACxD,MAAM,+BAA+B,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,SAAS,GAAG,MAAM,IAAA,gDAA4B,EAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAExE,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,iDAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QACnF,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAClE,aAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/B,aAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpC,aAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzC,aAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChC,aAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE3D,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,aAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,oEAAoE;QACpE,uCAAuC;QACvC,aAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/batched-migrations/batched-migration.d.ts

@@ -0,0 +1,79 @@
+import { z } from 'zod';
+export declare const BatchedMigrationStatusSchema: z.ZodEnum<["pending", "paused", "running", "finalizing", "failed", "succeeded"]>;
+export type BatchedMigrationStatus = z.infer<typeof BatchedMigrationStatusSchema>;
+export declare const BatchedMigrationRowSchema: z.ZodObject<{
+    id: z.ZodString;
+    project: z.ZodString;
+    filename: z.ZodString;
+    timestamp: z.ZodString;
+    batch_size: z.ZodNumber;
+    min_value: z.ZodBigInt;
+    max_value: z.ZodBigInt;
+    status: z.ZodEnum<["pending", "paused", "running", "finalizing", "failed", "succeeded"]>;
+    created_at: z.ZodDate;
+    updated_at: z.ZodDate;
+    started_at: z.ZodNullable<z.ZodDate>;
+}, "strip", z.ZodTypeAny, {
+    filename: string;
+    timestamp: string;
+    project: string;
+    status: "pending" | "paused" | "running" | "finalizing" | "failed" | "succeeded";
+    id: string;
+    batch_size: number;
+    min_value: bigint;
+    max_value: bigint;
+    created_at: Date;
+    updated_at: Date;
+    started_at: Date | null;
+}, {
+    filename: string;
+    timestamp: string;
+    project: string;
+    status: "pending" | "paused" | "running" | "finalizing" | "failed" | "succeeded";
+    id: string;
+    batch_size: number;
+    min_value: bigint;
+    max_value: bigint;
+    created_at: Date;
+    updated_at: Date;
+    started_at: Date | null;
+}>;
+export type BatchedMigrationRow = z.infer<typeof BatchedMigrationRowSchema>;
+export interface BatchedMigrationParameters {
+    min?: bigint | null;
+    max: bigint | null;
+    batchSize?: number;
+}
+export interface BatchedMigrationImplementation {
+    getParameters(): Promise<BatchedMigrationParameters>;
+    execute(start: bigint, end: bigint): Promise<void>;
+}
+/**
+ * Identity function that helps to write correct batched migrations.
+ */
+export declare function makeBatchedMigration<T extends BatchedMigrationImplementation>(fns: T): T;
+export declare function validateBatchedMigrationImplementation(fns: BatchedMigrationImplementation): asserts fns is BatchedMigrationImplementation;
+type NewBatchedMigration = Pick<BatchedMigrationRow, 'project' | 'filename' | 'timestamp' | 'batch_size' | 'min_value' | 'max_value' | 'status'>;
+/**
+ * Inserts a new batched migration. If one already exists for the given
+ * project/timestamp pair, returns null, otherwise returns the inserted row.
+ */
+export declare function insertBatchedMigration(migration: NewBatchedMigration): Promise<BatchedMigrationRow | null>;
+export declare function selectAllBatchedMigrations(project: string): Promise<{
+    filename: string;
+    timestamp: string;
+    project: string;
+    status: "pending" | "paused" | "running" | "finalizing" | "failed" | "succeeded";
+    id: string;
+    batch_size: number;
+    min_value: bigint;
+    max_value: bigint;
+    created_at: Date;
+    updated_at: Date;
+    started_at: Date | null;
+}[]>;
+export declare function selectBatchedMigration(project: string, id: string): Promise<BatchedMigrationRow>;
+export declare function selectBatchedMigrationForTimestamp(project: string, timestamp: string): Promise<BatchedMigrationRow>;
+export declare function updateBatchedMigrationStatus(id: string, status: BatchedMigrationStatus): Promise<BatchedMigrationRow>;
+export declare function retryFailedBatchedMigrationJobs(project: string, id: string): Promise<void>;
+export {};

package/dist/batched-migrations/batched-migration.js

@@ -0,0 +1,73 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.retryFailedBatchedMigrationJobs = exports.updateBatchedMigrationStatus = exports.selectBatchedMigrationForTimestamp = exports.selectBatchedMigration = exports.selectAllBatchedMigrations = exports.insertBatchedMigration = exports.validateBatchedMigrationImplementation = exports.makeBatchedMigration = exports.BatchedMigrationRowSchema = exports.BatchedMigrationStatusSchema = void 0;
+const postgres_1 = require("@prairielearn/postgres");
+const zod_1 = require("zod");
+const sql = (0, postgres_1.loadSqlEquiv)(__filename);
+exports.BatchedMigrationStatusSchema = zod_1.z.enum([
+    'pending',
+    'paused',
+    'running',
+    'finalizing',
+    'failed',
+    'succeeded',
+]);
+exports.BatchedMigrationRowSchema = zod_1.z.object({
+    id: zod_1.z.string(),
+    project: zod_1.z.string(),
+    filename: zod_1.z.string(),
+    timestamp: zod_1.z.string(),
+    batch_size: zod_1.z.number(),
+    min_value: zod_1.z.bigint({ coerce: true }),
+    max_value: zod_1.z.bigint({ coerce: true }),
+    status: exports.BatchedMigrationStatusSchema,
+    created_at: zod_1.z.date(),
+    updated_at: zod_1.z.date(),
+    started_at: zod_1.z.date().nullable(),
+});
+/**
+ * Identity function that helps to write correct batched migrations.
+ */
+function makeBatchedMigration(fns) {
+    validateBatchedMigrationImplementation(fns);
+    return fns;
+}
+exports.makeBatchedMigration = makeBatchedMigration;
+function validateBatchedMigrationImplementation(fns) {
+    if (typeof fns.getParameters !== 'function') {
+        throw new Error('getParameters() must be a function');
+    }
+    if (typeof fns.execute !== 'function') {
+        throw new Error('execute() must be a function');
+    }
+}
+exports.validateBatchedMigrationImplementation = validateBatchedMigrationImplementation;
+/**
+ * Inserts a new batched migration. If one already exists for the given
+ * project/timestamp pair, returns null, otherwise returns the inserted row.
+ */
+async function insertBatchedMigration(migration) {
+    return (0, postgres_1.queryValidatedZeroOrOneRow)(sql.insert_batched_migration, migration, exports.BatchedMigrationRowSchema);
+}
+exports.insertBatchedMigration = insertBatchedMigration;
+async function selectAllBatchedMigrations(project) {
+    return (0, postgres_1.queryValidatedRows)(sql.select_all_batched_migrations, { project }, exports.BatchedMigrationRowSchema);
+}
+exports.selectAllBatchedMigrations = selectAllBatchedMigrations;
+async function selectBatchedMigration(project, id) {
+    return (0, postgres_1.queryValidatedOneRow)(sql.select_batched_migration, { project, id }, exports.BatchedMigrationRowSchema);
+}
+exports.selectBatchedMigration = selectBatchedMigration;
+async function selectBatchedMigrationForTimestamp(project, timestamp) {
+    return (0, postgres_1.queryValidatedOneRow)(sql.select_batched_migration_for_timestamp, { project, timestamp }, exports.BatchedMigrationRowSchema);
+}
+exports.selectBatchedMigrationForTimestamp = selectBatchedMigrationForTimestamp;
+async function updateBatchedMigrationStatus(id, status) {
+    return (0, postgres_1.queryValidatedOneRow)(sql.update_batched_migration_status, { id, status }, exports.BatchedMigrationRowSchema);
+}
+exports.updateBatchedMigrationStatus = updateBatchedMigrationStatus;
+async function retryFailedBatchedMigrationJobs(project, id) {
+    await (0, postgres_1.queryAsync)(sql.retry_failed_jobs, { project, id });
+}
+exports.retryFailedBatchedMigrationJobs = retryFailedBatchedMigrationJobs;
+//# sourceMappingURL=batched-migration.js.map

package/dist/batched-migrations/batched-migration.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"batched-migration.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration.ts"],"names":[],"mappings":";;;AAAA,qDAMgC;AAChC,6BAAwB;AAExB,MAAM,GAAG,GAAG,IAAA,uBAAY,EAAC,UAAU,CAAC,CAAC;AAExB,QAAA,4BAA4B,GAAG,OAAC,CAAC,IAAI,CAAC;IACjD,SAAS;IACT,QAAQ;IACR,SAAS;IACT,YAAY;IACZ,QAAQ;IACR,WAAW;CACZ,CAAC,CAAC;AAGU,QAAA,yBAAyB,GAAG,OAAC,CAAC,MAAM,CAAC;IAChD,EAAE,EAAE,OAAC,CAAC,MAAM,EAAE;IACd,OAAO,EAAE,OAAC,CAAC,MAAM,EAAE;IACnB,QAAQ,EAAE,OAAC,CAAC,MAAM,EAAE;IACpB,SAAS,EAAE,OAAC,CAAC,MAAM,EAAE;IACrB,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE;IACtB,SAAS,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACrC,SAAS,EAAE,OAAC,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IACrC,MAAM,EAAE,oCAA4B;IACpC,UAAU,EAAE,OAAC,CAAC,IAAI,EAAE;IACpB,UAAU,EAAE,OAAC,CAAC,IAAI,EAAE;IACpB,UAAU,EAAE,OAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE;CAChC,CAAC,CAAC;AAcH;;GAEG;AACH,SAAgB,oBAAoB,CAA2C,GAAM;IACnF,sCAAsC,CAAC,GAAG,CAAC,CAAC;IAC5C,OAAO,GAAG,CAAC;AACb,CAAC;AAHD,oDAGC;AAED,SAAgB,sCAAsC,CACpD,GAAmC;IAEnC,IAAI,OAAO,GAAG,CAAC,aAAa,KAAK,UAAU,EAAE;QAC3C,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;KACvD;IACD,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,UAAU,EAAE;QACrC,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;KACjD;AACH,CAAC;AATD,wFASC;AAOD;;;GAGG;AACI,KAAK,UAAU,sBAAsB,CAC1C,SAA8B;IAE9B,OAAO,IAAA,qCAA0B,EAC/B,GAAG,CAAC,wBAAwB,EAC5B,SAAS,EACT,iCAAyB,CAC1B,CAAC;AACJ,CAAC;AARD,wDAQC;AAEM,KAAK,UAAU,0BAA0B,CAAC,OAAe;IAC9D,OAAO,IAAA,6BAAkB,EACvB,GAAG,CAAC,6BAA6B,EACjC,EAAE,OAAO,EAAE,EACX,iCAAyB,CAC1B,CAAC;AACJ,CAAC;AAND,gEAMC;AAEM,KAAK,UAAU,sBAAsB,CAC1C,OAAe,EACf,EAAU;IAEV,OAAO,IAAA,+BAAoB,EACzB,GAAG,CAAC,wBAAwB,EAC5B,EAAE,OAAO,EAAE,EAAE,EAAE,EACf,iCAAyB,CAC1B,CAAC;AACJ,CAAC;AATD,wDASC;AAEM,KAAK,UAAU,kCAAkC,CACtD,OAAe,EACf,SAAiB;IAEjB,OAAO,IAAA,+BAAoB,EACzB,GAAG,CAAC,sCAAsC,EAC1C,EAAE,OAAO,EAAE,SAAS,EAAE,EACtB,iCAAyB,CAC1B,CAAC;AACJ,CAAC;AATD,gFASC;AAEM,KAAK,UAAU,4BAA4B,CAChD,EAAU,EACV,MAA8B;IAE9B,OAAO,IAAA,+BAAoB,EACzB,GAAG,CAAC,+BAA+B,EACnC,EAAE,EAAE,EAAE,MAAM,EAAE,EACd,iCAAyB,CAC1B,CAAC;AACJ,CAAC;AATD,oEASC;AAEM,KAAK,UAAU,+BAA+B,CAAC,OAAe,EAAE,EAAU;IAC/E,MAAM,IAAA,qBAAU,EAAC,GAAG,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;AAC3D,CAAC;AAFD,0EAEC"}

package/dist/batched-migrations/batched-migration.sql

@@ -0,0 +1,95 @@
+-- BLOCK insert_batched_migration
+INSERT INTO
+  batched_migrations (
+    project,
+    filename,
+    timestamp,
+    batch_size,
+    min_value,
+    max_value,
+    status,
+    started_at
+  )
+VALUES
+  (
+    $project,
+    $filename,
+    $timestamp,
+    $batch_size,
+    $min_value,
+    $max_value,
+    $status,
+    -- If the migration is marked as already having succeeded, set `started_at`
+    -- since the migration did technically start.
+    CASE
+      WHEN $status::enum_batched_migration_status = 'succeeded' THEN CURRENT_TIMESTAMP
+      ELSE NULL
+    END
+  )
+ON CONFLICT DO NOTHING
+RETURNING
+  *;
+
+-- BLOCK select_all_batched_migrations
+SELECT
+  *
+FROM
+  batched_migrations
+WHERE
+  project = $project
+ORDER BY
+  id ASC;
+
+-- BLOCK select_batched_migration
+SELECT
+  *
+FROM
+  batched_migrations
+WHERE
+  project = $project
+  AND id = $id;
+
+-- BLOCK select_batched_migration_for_timestamp
+SELECT
+  *
+FROM
+  batched_migrations
+WHERE
+  project = $project
+  AND timestamp = $timestamp;
+
+-- BLOCK update_batched_migration_status
+UPDATE batched_migrations
+SET
+  status = $status,
+  updated_at = CURRENT_TIMESTAMP
+WHERE
+  id = $id
+RETURNING
+  *;
+
+-- BLOCK retry_failed_jobs
+WITH
+  updated_batched_migration AS (
+    UPDATE batched_migrations
+    SET
+      status = 'running',
+      started_at = CURRENT_TIMESTAMP,
+      updated_at = CURRENT_TIMESTAMP
+    WHERE
+      project = $project
+      AND id = $id
+    RETURNING
+      *
+  )
+UPDATE batched_migration_jobs
+SET
+  status = 'pending',
+  started_at = NULL,
+  finished_at = NULL,
+  updated_at = CURRENT_TIMESTAMP
+FROM
+  updated_batched_migration
+WHERE
+  batched_migration_id = updated_batched_migration.id
+  AND batched_migration_jobs.status = 'failed';

package/dist/batched-migrations/batched-migrations-runner.d.ts

@@ -0,0 +1,63 @@
+/// <reference types="node" />
+import EventEmitter from 'node:events';
+interface BatchedMigrationRunnerOptions {
+    project: string;
+    directories: string[];
+}
+interface BatchedMigrationStartOptions {
+    workDurationMs?: number;
+    sleepDurationMs?: number;
+}
+interface BatchedMigrationFinalizeOptions {
+    logProgress?: boolean;
+}
+export declare class BatchedMigrationsRunner extends EventEmitter {
+    private readonly options;
+    private readonly lockName;
+    private running;
+    private migrationFiles;
+    private abortController;
+    constructor(options: BatchedMigrationRunnerOptions);
+    private lockNameForTimestamp;
+    private getMigrationFiles;
+    private getMigrationForIdentifier;
+    /**
+     * Loads the implementation for the migration with the given identifier. The identifier
+     * must start with a 14-character timestamp. It may optionally be followed by
+     * an underscore with additional characters, which are ignored. These should
+     * typically be used to provide a human-readable name for the migration.
+     */
+    private loadMigrationImplementation;
+    enqueueBatchedMigration(identifier: string): Promise<void>;
+    finalizeBatchedMigration(identifier: string, options?: BatchedMigrationFinalizeOptions): Promise<void>;
+    start(options?: BatchedMigrationStartOptions): void;
+    loop({ workDurationMs, sleepDurationMs }: BatchedMigrationStartOptions): Promise<void>;
+    private getOrStartMigration;
+    maybePerformWork(durationMs: number): Promise<boolean>;
+    stop(): Promise<void>;
+}
+export declare function initBatchedMigrations(options: BatchedMigrationRunnerOptions): BatchedMigrationsRunner;
+export declare function startBatchedMigrations(options?: BatchedMigrationStartOptions): BatchedMigrationsRunner;
+export declare function stopBatchedMigrations(): Promise<void>;
+/**
+ * Given a batched migration identifier like `20230406184103_migration`,
+ * enqueues it for execution by creating a row in the `batched_migrations`
+ * table.
+ *
+ * Despite taking a full identifier, only the timestamp is used to uniquely
+ * identify the batched migration. The remaining part is just used to make
+ * calls more human-readable.
+ *
+ * @param identifier The identifier of the batched migration to enqueue.
+ */
+export declare function enqueueBatchedMigration(identifier: string): Promise<void>;
+/**
+ * Given a batched migration identifier like `20230406184103_migration`,
+ * synchronously runs it to completion. An error will be thrown if the final
+ * status of the migration is not `succeeded`.
+ *
+ * @param identifier The identifier of the batched migration to finalize.
+ * @param options Options for finalizing the batched migration.
+ */
+export declare function finalizeBatchedMigration(identifier: string, options?: BatchedMigrationFinalizeOptions): Promise<void>;
+export {};