@prairielearn/migrations 4.0.5 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/dist/batched-migrations/batched-migration-runner.test.d.ts.map +1 -1
- package/dist/batched-migrations/batched-migration-runner.test.js +3 -2
- package/dist/batched-migrations/batched-migration-runner.test.js.map +1 -1
- package/package.json +14 -11
- package/src/batched-migrations/batched-migration-runner.test.ts +3 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,30 @@
|
|
|
1
1
|
# @prairielearn/migrations
|
|
2
2
|
|
|
3
|
+
## 5.0.1
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- 8bdf6ea: Upgrade all JavaScript dependencies
|
|
8
|
+
- Updated dependencies [8bdf6ea]
|
|
9
|
+
- @prairielearn/named-locks@4.0.1
|
|
10
|
+
- @prairielearn/postgres@5.0.2
|
|
11
|
+
- @prairielearn/logger@3.0.1
|
|
12
|
+
- @prairielearn/error@3.0.2
|
|
13
|
+
|
|
14
|
+
## 5.0.0
|
|
15
|
+
|
|
16
|
+
### Major Changes
|
|
17
|
+
|
|
18
|
+
- 3914bb4: Upgrade to Node 24
|
|
19
|
+
|
|
20
|
+
### Patch Changes
|
|
21
|
+
|
|
22
|
+
- Updated dependencies [3914bb4]
|
|
23
|
+
- @prairielearn/named-locks@4.0.0
|
|
24
|
+
- @prairielearn/postgres@5.0.0
|
|
25
|
+
- @prairielearn/logger@3.0.0
|
|
26
|
+
- @prairielearn/error@3.0.0
|
|
27
|
+
|
|
3
28
|
## 4.0.5
|
|
4
29
|
|
|
5
30
|
### Patch Changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batched-migration-runner.test.d.ts","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { execute, makePostgresTestUtils, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await execute(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n { batched_migration_id: migrationId },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n const poolConfig = await postgresTestUtils.createDatabase();\n await namedLocks.init(poolConfig, (err) => {\n throw err;\n });\n await init({ directories: [SCHEMA_MIGRATIONS_PATH], project: 'prairielearn_migrations' });\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n assert.equal(jobData.error.data.start, job.min_value
|
|
1
|
+
{"version":3,"file":"batched-migration-runner.test.d.ts","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { execute, makePostgresTestUtils, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await execute(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n { batched_migration_id: migrationId },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n const poolConfig = await postgresTestUtils.createDatabase();\n await namedLocks.init(poolConfig, (err) => {\n throw err;\n });\n await init({ directories: [SCHEMA_MIGRATIONS_PATH], project: 'prairielearn_migrations' });\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n // serialize-error converts BigInts to strings with 'n' suffix\n assert.equal(jobData.error.data.start, `${job.min_value}n`);\n assert.equal(jobData.error.data.end, `${job.max_value}n`);\n });\n\n const failedMigration = await getBatchedMigration(migration.id);\n assert.equal(failedMigration.status, 'failed');\n\n // Retry the failed jobs; ensure they succeed this time.\n await resetFailedBatchedMigrationJobs(migration.id);\n migration = await updateBatchedMigrationStatus(migration.id, 'running');\n\n migrationImplementation.setFailingIds([]);\n const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);\n await retryRunner.run();\n\n const finalJobs = await getBatchedMigrationJobs(migration.id);\n const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');\n const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');\n const retriedJobs = finalJobs.filter((job) => job.attempts === 2);\n assert.lengthOf(finalJobs, 10);\n assert.lengthOf(finalFailedJobs, 0);\n assert.lengthOf(finalSuccessfulJobs, 10);\n assert.lengthOf(retriedJobs, 2);\n assert.isTrue(finalJobs.every((job) => job.data === null));\n\n migration = await getBatchedMigration(migration.id);\n assert.equal(migration.status, 'succeeded');\n\n // The runner should have run only the previously failed jobs, which\n // works out to 2 additional execution.\n assert.equal(migrationImplementation.executionCount, 12);\n });\n});\n"]}
|
|
@@ -128,8 +128,9 @@ describe('BatchedMigrationExecutor', () => {
|
|
|
128
128
|
assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);
|
|
129
129
|
assert.equal(jobData.error.name, 'Error');
|
|
130
130
|
assert.equal(jobData.error.message, 'Execution failure');
|
|
131
|
-
|
|
132
|
-
assert.equal(jobData.error.data.
|
|
131
|
+
// serialize-error converts BigInts to strings with 'n' suffix
|
|
132
|
+
assert.equal(jobData.error.data.start, `${job.min_value}n`);
|
|
133
|
+
assert.equal(jobData.error.data.end, `${job.max_value}n`);
|
|
133
134
|
});
|
|
134
135
|
const failedMigration = await getBatchedMigration(migration.id);
|
|
135
136
|
assert.equal(failedMigration.status, 'failed');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batched-migration-runner.test.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE/E,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAE7F,OAAO,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;IAC9C,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AAEH,SAAS,sBAAsB,GAAG;IAChC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,OAAO,oBAAoB,CAAC;QAC1B,KAAK,CAAC,aAAa,GAAG;YACpB,OAAO;gBACL,GAAG,EAAE,EAAE;gBACP,GAAG,EAAE,MAAM;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;QAAA,CACH;QACD,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,GAAW,EAAE;YACxC,cAAc,IAAI,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;YACrE,IAAI,UAAU,EAAE,CAAC;gBACf,mEAAmE;gBACnE,iEAAiE;gBACjE,mCAAmC;gBACnC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAChF,CAAC;QAAA,CACF;QACD,aAAa,CAAC,GAAa,EAAE;YAC3B,UAAU,GAAG,GAAG,CAAC;QAAA,CAClB;QACD,IAAI,cAAc,GAAG;YACnB,OAAO,cAAc,CAAC;QAAA,CACvB;KACF,CAAC,CAAC;AAAA,CACJ;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE;IACtD,OAAO,MAAM,QAAQ,CACnB,kDAAkD,EAClD,EAAE,EAAE,EAAE,WAAW,EAAE,EACnB,yBAAyB,CAC1B,CAAC;AAAA,CACH;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB,EAAE;IAC1D,OAAO,MAAM,SAAS,CACpB,0GAA0G,EAC1G,EAAE,oBAAoB,EAAE,WAAW,EAAE,EACrC,4BAA4B,CAC7B,CAAC;AAAA,CACH;AAED,KAAK,UAAU,+BAA+B,CAAC,WAAmB,EAAE;IAClE,MAAM,OAAO,CACX,+JAA+J,EAC/J,EAAE,oBAAoB,EAAE,WAAW,EAAE,CACtC,CAAC;AAAA,CACH;AAED,KAAK,UAAU,0BAA0B,GAAG;IAC1C,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC;QAC7C,OAAO,EAAE,MAAM;QACf,QAAQ,EAAE,wCAAwC;QAClD,SAAS,EAAE,gBAAgB;QAC3B,UAAU,EAAE,UAAU,CAAC,SAAS;QAChC,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AAAA,CAClB;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE,CAAC;IACzC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;YACzC,MAAM,GAAG,CAAC;QAAA,CACX,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,EAAE,WAAW,EAAE,CAAC,sBAAsB,CAAC,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;IAAA,CAC3F,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAAA,CACzC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;IAAA,CACxC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAChF,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAAA,CACzD,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAAA,CAClD,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAEnD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,uBAAuB,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAW,CAAC;YAChC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QAAA,CAChE,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE/C,wDAAwD;QACxD,MAAM,+BAA+B,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAExE,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QACnF,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE3D,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,oEAAoE;QACpE,uCAAuC;QACvC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAAA,CAC1D,CAAC,CAAC;AAAA,CACJ,CAAC,CAAC","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { execute, makePostgresTestUtils, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await execute(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n { batched_migration_id: migrationId },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n const poolConfig = await postgresTestUtils.createDatabase();\n await namedLocks.init(poolConfig, (err) => {\n throw err;\n });\n await init({ directories: [SCHEMA_MIGRATIONS_PATH], project: 'prairielearn_migrations' });\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n assert.equal(jobData.error.data.start, job.min_value.toString());\n assert.equal(jobData.error.data.end, job.max_value.toString());\n });\n\n const failedMigration = await getBatchedMigration(migration.id);\n assert.equal(failedMigration.status, 'failed');\n\n // Retry the failed jobs; ensure they succeed this time.\n await resetFailedBatchedMigrationJobs(migration.id);\n migration = await updateBatchedMigrationStatus(migration.id, 'running');\n\n migrationImplementation.setFailingIds([]);\n const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);\n await retryRunner.run();\n\n const finalJobs = await getBatchedMigrationJobs(migration.id);\n const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');\n const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');\n const retriedJobs = finalJobs.filter((job) => job.attempts === 2);\n assert.lengthOf(finalJobs, 10);\n assert.lengthOf(finalFailedJobs, 0);\n assert.lengthOf(finalSuccessfulJobs, 10);\n assert.lengthOf(retriedJobs, 2);\n assert.isTrue(finalJobs.every((job) => job.data === null));\n\n migration = await getBatchedMigration(migration.id);\n assert.equal(migration.status, 'succeeded');\n\n // The runner should have run only the previously failed jobs, which\n // works out to 2 additional execution.\n assert.equal(migrationImplementation.executionCount, 12);\n });\n});\n"]}
|
|
1
|
+
{"version":3,"file":"batched-migration-runner.test.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE/E,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,qBAAqB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAE7F,OAAO,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;IAC9C,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AAEH,SAAS,sBAAsB,GAAG;IAChC,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,OAAO,oBAAoB,CAAC;QAC1B,KAAK,CAAC,aAAa,GAAG;YACpB,OAAO;gBACL,GAAG,EAAE,EAAE;gBACP,GAAG,EAAE,MAAM;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;QAAA,CACH;QACD,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,GAAW,EAAE;YACxC,cAAc,IAAI,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;YACrE,IAAI,UAAU,EAAE,CAAC;gBACf,mEAAmE;gBACnE,iEAAiE;gBACjE,mCAAmC;gBACnC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAChF,CAAC;QAAA,CACF;QACD,aAAa,CAAC,GAAa,EAAE;YAC3B,UAAU,GAAG,GAAG,CAAC;QAAA,CAClB;QACD,IAAI,cAAc,GAAG;YACnB,OAAO,cAAc,CAAC;QAAA,CACvB;KACF,CAAC,CAAC;AAAA,CACJ;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB,EAAE;IACtD,OAAO,MAAM,QAAQ,CACnB,kDAAkD,EAClD,EAAE,EAAE,EAAE,WAAW,EAAE,EACnB,yBAAyB,CAC1B,CAAC;AAAA,CACH;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB,EAAE;IAC1D,OAAO,MAAM,SAAS,CACpB,0GAA0G,EAC1G,EAAE,oBAAoB,EAAE,WAAW,EAAE,EACrC,4BAA4B,CAC7B,CAAC;AAAA,CACH;AAED,KAAK,UAAU,+BAA+B,CAAC,WAAmB,EAAE;IAClE,MAAM,OAAO,CACX,+JAA+J,EAC/J,EAAE,oBAAoB,EAAE,WAAW,EAAE,CACtC,CAAC;AAAA,CACH;AAED,KAAK,UAAU,0BAA0B,GAAG;IAC1C,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC;QAC7C,OAAO,EAAE,MAAM;QACf,QAAQ,EAAE,wCAAwC;QAClD,SAAS,EAAE,gBAAgB;QAC3B,UAAU,EAAE,UAAU,CAAC,SAAS;QAChC,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AAAA,CAClB;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE,CAAC;IACzC,SAAS,CAAC,KAAK,IAAI,EAAE,CAAC;QACpB,MAAM,UAAU,GAAG,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QAC5D,MAAM,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC;YACzC,MAAM,GAAG,CAAC;QAAA,CACX,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,EAAE,WAAW,EAAE,CAAC,sBAAsB,CAAC,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;IAAA,CAC3F,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE,CAAC;QACrB,MAAM,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAAA,CACzC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE,CAAC;QACnB,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;IAAA,CACxC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAChF,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAAA,CACzD,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAAA,CAClD,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1C,IAAI,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAEnD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,uBAAuB,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC;YAC1B,MAAM,OAAO,GAAG,GAAG,CAAC,IAAW,CAAC;YAChC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;YACzD,8DAA8D;YAC9D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC;YAC5D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC;QAAA,CAC3D,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE/C,wDAAwD;QACxD,MAAM,+BAA+B,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAExE,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QACnF,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE3D,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,oEAAoE;QACpE,uCAAuC;QACvC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAAA,CAC1D,CAAC,CAAC;AAAA,CACJ,CAAC,CAAC","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { execute, makePostgresTestUtils, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await execute(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n { batched_migration_id: migrationId },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n const poolConfig = await postgresTestUtils.createDatabase();\n await namedLocks.init(poolConfig, (err) => {\n throw err;\n });\n await init({ directories: [SCHEMA_MIGRATIONS_PATH], project: 'prairielearn_migrations' });\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n // serialize-error converts BigInts to strings with 'n' suffix\n assert.equal(jobData.error.data.start, `${job.min_value}n`);\n assert.equal(jobData.error.data.end, `${job.max_value}n`);\n });\n\n const failedMigration = await getBatchedMigration(migration.id);\n assert.equal(failedMigration.status, 'failed');\n\n // Retry the failed jobs; ensure they succeed this time.\n await resetFailedBatchedMigrationJobs(migration.id);\n migration = await updateBatchedMigrationStatus(migration.id, 'running');\n\n migrationImplementation.setFailingIds([]);\n const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);\n await retryRunner.run();\n\n const finalJobs = await getBatchedMigrationJobs(migration.id);\n const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');\n const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');\n const retriedJobs = finalJobs.filter((job) => job.attempts === 2);\n assert.lengthOf(finalJobs, 10);\n assert.lengthOf(finalFailedJobs, 0);\n assert.lengthOf(finalSuccessfulJobs, 10);\n assert.lengthOf(retriedJobs, 2);\n assert.isTrue(finalJobs.every((job) => job.data === null));\n\n migration = await getBatchedMigration(migration.id);\n assert.equal(migration.status, 'succeeded');\n\n // The runner should have run only the previously failed jobs, which\n // works out to 2 additional execution.\n assert.equal(migrationImplementation.executionCount, 12);\n });\n});\n"]}
|
package/package.json
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prairielearn/migrations",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.0.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "https://github.com/PrairieLearn/PrairieLearn.git",
|
|
8
8
|
"directory": "packages/migrations"
|
|
9
9
|
},
|
|
10
|
+
"engines": {
|
|
11
|
+
"node": ">=24.0.0"
|
|
12
|
+
},
|
|
10
13
|
"main": "./dist/index.js",
|
|
11
14
|
"scripts": {
|
|
12
15
|
"build": "tsgo && tscp",
|
|
@@ -14,24 +17,24 @@
|
|
|
14
17
|
"test": "vitest run --coverage"
|
|
15
18
|
},
|
|
16
19
|
"dependencies": {
|
|
17
|
-
"@prairielearn/error": "^
|
|
18
|
-
"@prairielearn/logger": "^
|
|
19
|
-
"@prairielearn/named-locks": "^
|
|
20
|
-
"@prairielearn/postgres": "^
|
|
20
|
+
"@prairielearn/error": "^3.0.2",
|
|
21
|
+
"@prairielearn/logger": "^3.0.1",
|
|
22
|
+
"@prairielearn/named-locks": "^4.0.1",
|
|
23
|
+
"@prairielearn/postgres": "^5.0.2",
|
|
21
24
|
"fs-extra": "^11.3.3",
|
|
22
|
-
"serialize-error": "^
|
|
25
|
+
"serialize-error": "^13.0.1",
|
|
23
26
|
"zod": "^3.25.76"
|
|
24
27
|
},
|
|
25
28
|
"devDependencies": {
|
|
26
|
-
"@prairielearn/tsconfig": "^
|
|
29
|
+
"@prairielearn/tsconfig": "^2.0.0",
|
|
27
30
|
"@types/fs-extra": "^11.0.4",
|
|
28
|
-
"@types/node": "^
|
|
29
|
-
"@typescript/native-preview": "^7.0.0-dev.
|
|
30
|
-
"@vitest/coverage-v8": "^4.0.
|
|
31
|
+
"@types/node": "^24.10.9",
|
|
32
|
+
"@typescript/native-preview": "^7.0.0-dev.20260130.1",
|
|
33
|
+
"@vitest/coverage-v8": "^4.0.18",
|
|
31
34
|
"tmp-promise": "^3.0.3",
|
|
32
35
|
"tsx": "^4.21.0",
|
|
33
36
|
"typescript": "^5.9.3",
|
|
34
37
|
"typescript-cp": "^0.1.9",
|
|
35
|
-
"vitest": "^4.0.
|
|
38
|
+
"vitest": "^4.0.18"
|
|
36
39
|
}
|
|
37
40
|
}
|
|
@@ -168,8 +168,9 @@ describe('BatchedMigrationExecutor', () => {
|
|
|
168
168
|
assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);
|
|
169
169
|
assert.equal(jobData.error.name, 'Error');
|
|
170
170
|
assert.equal(jobData.error.message, 'Execution failure');
|
|
171
|
-
|
|
172
|
-
assert.equal(jobData.error.data.
|
|
171
|
+
// serialize-error converts BigInts to strings with 'n' suffix
|
|
172
|
+
assert.equal(jobData.error.data.start, `${job.min_value}n`);
|
|
173
|
+
assert.equal(jobData.error.data.end, `${job.max_value}n`);
|
|
173
174
|
});
|
|
174
175
|
|
|
175
176
|
const failedMigration = await getBatchedMigration(migration.id);
|