@prairielearn/migrations 3.0.20 → 3.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/dist/batched-migrations/batched-migration-runner.js +1 -3
- package/dist/batched-migrations/batched-migration-runner.js.map +1 -1
- package/dist/batched-migrations/batched-migration-runner.test.js +1 -3
- package/dist/batched-migrations/batched-migration-runner.test.js.map +1 -1
- package/package.json +8 -8
- package/src/batched-migrations/batched-migration-runner.test.ts +1 -3
- package/src/batched-migrations/batched-migration-runner.ts +1 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# @prairielearn/migrations
|
|
2
2
|
|
|
3
|
+
## 3.0.22
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- b55261c: Upgrade to TypeScript 5.9
|
|
8
|
+
- Updated dependencies [b55261c]
|
|
9
|
+
- @prairielearn/named-locks@3.0.22
|
|
10
|
+
- @prairielearn/postgres@4.0.1
|
|
11
|
+
- @prairielearn/logger@2.0.19
|
|
12
|
+
- @prairielearn/error@2.0.18
|
|
13
|
+
|
|
14
|
+
## 3.0.21
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- 23adb05: Upgrade all JavaScript dependencies
|
|
19
|
+
- Updated dependencies [cd5ed49]
|
|
20
|
+
- Updated dependencies [2c19e43]
|
|
21
|
+
- Updated dependencies [23adb05]
|
|
22
|
+
- @prairielearn/postgres@4.0.0
|
|
23
|
+
- @prairielearn/named-locks@3.0.21
|
|
24
|
+
- @prairielearn/logger@2.0.18
|
|
25
|
+
- @prairielearn/error@2.0.17
|
|
26
|
+
|
|
3
27
|
## 3.0.20
|
|
4
28
|
|
|
5
29
|
### Patch Changes
|
|
@@ -28,9 +28,7 @@ export class BatchedMigrationRunner {
|
|
|
28
28
|
return await queryRow(sql.batched_migration_has_failed_jobs, { batched_migration_id: migration.id }, z.boolean());
|
|
29
29
|
}
|
|
30
30
|
async refreshMigrationStatus(migration) {
|
|
31
|
-
this.migrationStatus = await queryRow(sql.get_migration_status, {
|
|
32
|
-
id: migration.id,
|
|
33
|
-
}, BatchedMigrationStatusSchema);
|
|
31
|
+
this.migrationStatus = await queryRow(sql.get_migration_status, { id: migration.id }, BatchedMigrationStatusSchema);
|
|
34
32
|
}
|
|
35
33
|
async finishRunningMigration(migration) {
|
|
36
34
|
// Safety check: if there are any pending jobs, don't mark this
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EAEL,4BAA4B,GAE7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAIL,4BAA4B,EAC5B,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB;YACE,EAAE,EAAE,SAAS,CAAC,EAAE;SACjB,EACD,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryOptionalRow, queryRow } from '@prairielearn/postgres';\n\nimport {\n type BatchedMigrationJobRow,\n BatchedMigrationJobRowSchema,\n type BatchedMigrationJobStatus,\n} from './batched-migration-job.js';\nimport {\n type BatchedMigrationImplementation,\n type BatchedMigrationRow,\n type BatchedMigrationStatus,\n BatchedMigrationStatusSchema,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n private options: BatchedMigrationRunnerOptions;\n private migration: BatchedMigrationRow;\n private migrationImplementation: BatchedMigrationImplementation;\n private migrationStatus: BatchedMigrationStatus;\n\n constructor(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n options: BatchedMigrationRunnerOptions = {},\n ) {\n this.options = options;\n this.migration = migration;\n this.migrationImplementation = migrationImplementation;\n this.migrationStatus = migration.status;\n }\n\n private log(message: string, ...meta: any[]) {\n if (this.options.logProgress) {\n logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n }\n }\n\n private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_incomplete_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_failed_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n this.migrationStatus = await queryRow(\n sql.get_migration_status,\n {\n id: migration.id,\n },\n BatchedMigrationStatusSchema,\n );\n }\n\n private async finishRunningMigration(migration: BatchedMigrationRow) {\n // Safety check: if there are any pending jobs, don't mark this\n // migration as finished.\n if (await this.hasIncompleteJobs(migration)) {\n this.log('Incomplete jobs found, not marking as finished');\n return;\n }\n\n const hasFailedJobs = await this.hasFailedJobs(migration);\n const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n await updateBatchedMigrationStatus(migration.id, finalStatus);\n this.log(`Finished with status '${finalStatus}'`);\n }\n\n private async getNextBatchBounds(\n migration: BatchedMigrationRow,\n ): Promise<null | [bigint, bigint]> {\n const lastJob = await queryOptionalRow(\n sql.select_last_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n\n const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n if (nextMin > migration.max_value) return null;\n\n let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n return [nextMin, nextMax];\n }\n\n private async startJob(job: BatchedMigrationJobRow) {\n await queryAsync(sql.start_batched_migration_job, { id: job.id });\n const jobRange = `[${job.min_value}, ${job.max_value}]`;\n const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n }\n\n private serializeJobData(data: unknown) {\n if (data == null) return null;\n\n // Return JSON-stringified data. Convert BigInts to strings.\n return JSON.stringify(data, (_key, value) => {\n if (typeof value === 'bigint') return value.toString();\n return value;\n });\n }\n\n private async finishJob(\n job: BatchedMigrationJobRow,\n status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n data?: unknown,\n ) {\n await queryAsync(sql.finish_batched_migration_job, {\n id: job.id,\n status,\n data: this.serializeJobData(data),\n });\n this.log(`Job ${job.id} finished with status '${status}'`);\n }\n\n private async getOrCreateNextMigrationJob(\n migration: BatchedMigrationRow,\n ): Promise<BatchedMigrationJobRow | null> {\n const nextBatchBounds = await this.getNextBatchBounds(migration);\n if (nextBatchBounds) {\n return await queryRow(\n sql.insert_batched_migration_job,\n {\n batched_migration_id: migration.id,\n min_value: nextBatchBounds[0],\n max_value: nextBatchBounds[1],\n },\n BatchedMigrationJobRowSchema,\n );\n } else {\n // Pick up any old pending jobs from this migration. These will only exist if\n // an admin manually elected to retry all failed jobs; we'll never automatically\n // transition failed jobs back to pending.\n return await queryOptionalRow(\n sql.select_first_pending_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n }\n }\n\n private async runMigrationJob(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n ) {\n const nextJob = await this.getOrCreateNextMigrationJob(migration);\n if (nextJob) {\n await this.startJob(nextJob);\n\n let error = null;\n try {\n // We'll only handle errors thrown by the migration itself. If any of\n // our own execution machinery throws an error, we'll let it bubble up.\n await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n } catch (err) {\n error = err;\n }\n\n if (error) {\n logger.error(\n `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n error,\n );\n await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n } else {\n await this.finishJob(nextJob, 'succeeded');\n }\n } else {\n await this.finishRunningMigration(migration);\n }\n }\n\n async run({\n signal,\n iterations,\n durationMs,\n }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n let iterationCount = 0;\n const endTime = durationMs ? Date.now() + durationMs : null;\n while (\n !signal?.aborted &&\n (iterations ? iterationCount < iterations : true) &&\n (endTime ? Date.now() < endTime : true) &&\n (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n ) {\n await this.runMigrationJob(this.migration, this.migrationImplementation);\n iterationCount += 1;\n // Always refresh the status so we can detect if the migration was marked\n // as paused by another process.\n await this.refreshMigrationStatus(this.migration);\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EAEL,4BAA4B,GAE7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAIL,4BAA4B,EAC5B,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB,EAAE,EAAE,EAAE,SAAS,CAAC,EAAE,EAAE,EACpB,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryOptionalRow, queryRow } from '@prairielearn/postgres';\n\nimport {\n type BatchedMigrationJobRow,\n BatchedMigrationJobRowSchema,\n type BatchedMigrationJobStatus,\n} from './batched-migration-job.js';\nimport {\n type BatchedMigrationImplementation,\n type BatchedMigrationRow,\n type BatchedMigrationStatus,\n BatchedMigrationStatusSchema,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n private options: BatchedMigrationRunnerOptions;\n private migration: BatchedMigrationRow;\n private migrationImplementation: BatchedMigrationImplementation;\n private migrationStatus: BatchedMigrationStatus;\n\n constructor(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n options: BatchedMigrationRunnerOptions = {},\n ) {\n this.options = options;\n this.migration = migration;\n this.migrationImplementation = migrationImplementation;\n this.migrationStatus = migration.status;\n }\n\n private log(message: string, ...meta: any[]) {\n if (this.options.logProgress) {\n logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n }\n }\n\n private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_incomplete_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_failed_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n this.migrationStatus = await queryRow(\n sql.get_migration_status,\n { id: migration.id },\n BatchedMigrationStatusSchema,\n );\n }\n\n private async finishRunningMigration(migration: BatchedMigrationRow) {\n // Safety check: if there are any pending jobs, don't mark this\n // migration as finished.\n if (await this.hasIncompleteJobs(migration)) {\n this.log('Incomplete jobs found, not marking as finished');\n return;\n }\n\n const hasFailedJobs = await this.hasFailedJobs(migration);\n const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n await updateBatchedMigrationStatus(migration.id, finalStatus);\n this.log(`Finished with status '${finalStatus}'`);\n }\n\n private async getNextBatchBounds(\n migration: BatchedMigrationRow,\n ): Promise<null | [bigint, bigint]> {\n const lastJob = await queryOptionalRow(\n sql.select_last_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n\n const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n if (nextMin > migration.max_value) return null;\n\n let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n return [nextMin, nextMax];\n }\n\n private async startJob(job: BatchedMigrationJobRow) {\n await queryAsync(sql.start_batched_migration_job, { id: job.id });\n const jobRange = `[${job.min_value}, ${job.max_value}]`;\n const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n }\n\n private serializeJobData(data: unknown) {\n if (data == null) return null;\n\n // Return JSON-stringified data. Convert BigInts to strings.\n return JSON.stringify(data, (_key, value) => {\n if (typeof value === 'bigint') return value.toString();\n return value;\n });\n }\n\n private async finishJob(\n job: BatchedMigrationJobRow,\n status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n data?: unknown,\n ) {\n await queryAsync(sql.finish_batched_migration_job, {\n id: job.id,\n status,\n data: this.serializeJobData(data),\n });\n this.log(`Job ${job.id} finished with status '${status}'`);\n }\n\n private async getOrCreateNextMigrationJob(\n migration: BatchedMigrationRow,\n ): Promise<BatchedMigrationJobRow | null> {\n const nextBatchBounds = await this.getNextBatchBounds(migration);\n if (nextBatchBounds) {\n return await queryRow(\n sql.insert_batched_migration_job,\n {\n batched_migration_id: migration.id,\n min_value: nextBatchBounds[0],\n max_value: nextBatchBounds[1],\n },\n BatchedMigrationJobRowSchema,\n );\n } else {\n // Pick up any old pending jobs from this migration. These will only exist if\n // an admin manually elected to retry all failed jobs; we'll never automatically\n // transition failed jobs back to pending.\n return await queryOptionalRow(\n sql.select_first_pending_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n }\n }\n\n private async runMigrationJob(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n ) {\n const nextJob = await this.getOrCreateNextMigrationJob(migration);\n if (nextJob) {\n await this.startJob(nextJob);\n\n let error = null;\n try {\n // We'll only handle errors thrown by the migration itself. If any of\n // our own execution machinery throws an error, we'll let it bubble up.\n await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n } catch (err) {\n error = err;\n }\n\n if (error) {\n logger.error(\n `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n error,\n );\n await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n } else {\n await this.finishJob(nextJob, 'succeeded');\n }\n } else {\n await this.finishRunningMigration(migration);\n }\n }\n\n async run({\n signal,\n iterations,\n durationMs,\n }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n let iterationCount = 0;\n const endTime = durationMs ? Date.now() + durationMs : null;\n while (\n !signal?.aborted &&\n (iterations ? iterationCount < iterations : true) &&\n (endTime ? Date.now() < endTime : true) &&\n (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n ) {\n await this.runMigrationJob(this.migration, this.migrationImplementation);\n iterationCount += 1;\n // Always refresh the status so we can detect if the migration was marked\n // as paused by another process.\n await this.refreshMigrationStatus(this.migration);\n }\n }\n}\n"]}
|
|
@@ -45,9 +45,7 @@ async function getBatchedMigrationJobs(migrationId) {
|
|
|
45
45
|
return await queryRows('SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;', { batched_migration_id: migrationId }, BatchedMigrationJobRowSchema);
|
|
46
46
|
}
|
|
47
47
|
async function resetFailedBatchedMigrationJobs(migrationId) {
|
|
48
|
-
await queryAsync("UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'", {
|
|
49
|
-
batched_migration_id: migrationId,
|
|
50
|
-
});
|
|
48
|
+
await queryAsync("UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'", { batched_migration_id: migrationId });
|
|
51
49
|
}
|
|
52
50
|
async function insertTestBatchedMigration() {
|
|
53
51
|
const migrationImplementation = makeTestBatchMigration();
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batched-migration-runner.test.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE/E,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEhG,OAAO,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;IAC9C,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AAEH,SAAS,sBAAsB;IAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,OAAO,oBAAoB,CAAC;QAC1B,KAAK,CAAC,aAAa;YACjB,OAAO;gBACL,GAAG,EAAE,EAAE;gBACP,GAAG,EAAE,MAAM;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,GAAW;YACtC,cAAc,IAAI,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;YACrE,IAAI,UAAU,EAAE,CAAC;gBACf,mEAAmE;gBACnE,iEAAiE;gBACjE,mCAAmC;gBACnC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;QACD,aAAa,CAAC,GAAa;YACzB,UAAU,GAAG,GAAG,CAAC;QACnB,CAAC;QACD,IAAI,cAAc;YAChB,OAAO,cAAc,CAAC;QACxB,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB;IACpD,OAAO,MAAM,QAAQ,CACnB,kDAAkD,EAClD,EAAE,EAAE,EAAE,WAAW,EAAE,EACnB,yBAAyB,CAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB;IACxD,OAAO,MAAM,SAAS,CACpB,0GAA0G,EAC1G,EAAE,oBAAoB,EAAE,WAAW,EAAE,EACrC,4BAA4B,CAC7B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,+BAA+B,CAAC,WAAmB;IAChE,MAAM,UAAU,CACd,+JAA+J,EAC/J;QACE,oBAAoB,EAAE,WAAW;KAClC,CACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,0BAA0B;IACvC,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC;QAC7C,OAAO,EAAE,MAAM;QACf,QAAQ,EAAE,wCAAwC;QAClD,SAAS,EAAE,gBAAgB;QAC3B,UAAU,EAAE,UAAU,CAAC,SAAS;QAChC,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QACzC,MAAM,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,CAAC,sBAAsB,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE;QAClB,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAChF,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,IAAI,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAEnD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,uBAAuB,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,GAAG,CAAC,IAAW,CAAC;YAChC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE/C,wDAAwD;QACxD,MAAM,+BAA+B,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAExE,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QACnF,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE3D,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,oEAAoE;QACpE,uCAAuC;QACvC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { makePostgresTestUtils, queryAsync, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await queryAsync(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n {\n batched_migration_id: migrationId,\n },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n await postgresTestUtils.createDatabase();\n await namedLocks.init(postgresTestUtils.getPoolConfig(), (err) => {\n throw err;\n });\n await init([SCHEMA_MIGRATIONS_PATH], 'prairielearn_migrations');\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n assert.equal(jobData.error.data.start, job.min_value.toString());\n assert.equal(jobData.error.data.end, job.max_value.toString());\n });\n\n const failedMigration = await getBatchedMigration(migration.id);\n assert.equal(failedMigration.status, 'failed');\n\n // Retry the failed jobs; ensure they succeed this time.\n await resetFailedBatchedMigrationJobs(migration.id);\n migration = await updateBatchedMigrationStatus(migration.id, 'running');\n\n migrationImplementation.setFailingIds([]);\n const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);\n await retryRunner.run();\n\n const finalJobs = await getBatchedMigrationJobs(migration.id);\n const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');\n const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');\n const retriedJobs = finalJobs.filter((job) => job.attempts === 2);\n assert.lengthOf(finalJobs, 10);\n assert.lengthOf(finalFailedJobs, 0);\n assert.lengthOf(finalSuccessfulJobs, 10);\n assert.lengthOf(retriedJobs, 2);\n assert.isTrue(finalJobs.every((job) => job.data === null));\n\n migration = await getBatchedMigration(migration.id);\n assert.equal(migration.status, 'succeeded');\n\n // The runner should have run only the previously failed jobs, which\n // works out to 2 additional execution.\n assert.equal(migrationImplementation.executionCount, 12);\n });\n});\n"]}
|
|
1
|
+
{"version":3,"file":"batched-migration-runner.test.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAE/E,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAEhG,OAAO,EAAE,sBAAsB,EAAE,IAAI,EAAE,MAAM,aAAa,CAAC;AAE3D,OAAO,EAAE,4BAA4B,EAAE,MAAM,4BAA4B,CAAC;AAC1E,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EACzB,sBAAsB,EACtB,oBAAoB,EACpB,4BAA4B,GAC7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,iBAAiB,GAAG,qBAAqB,CAAC;IAC9C,QAAQ,EAAE,yBAAyB;CACpC,CAAC,CAAC;AAEH,SAAS,sBAAsB;IAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,UAAU,GAAa,EAAE,CAAC;IAE9B,OAAO,oBAAoB,CAAC;QAC1B,KAAK,CAAC,aAAa;YACjB,OAAO;gBACL,GAAG,EAAE,EAAE;gBACP,GAAG,EAAE,MAAM;gBACX,SAAS,EAAE,IAAI;aAChB,CAAC;QACJ,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,KAAa,EAAE,GAAW;YACtC,cAAc,IAAI,CAAC,CAAC;YACpB,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,IAAI,KAAK,IAAI,EAAE,IAAI,GAAG,CAAC,CAAC;YACrE,IAAI,UAAU,EAAE,CAAC;gBACf,mEAAmE;gBACnE,iEAAiE;gBACjE,mCAAmC;gBACnC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAChF,CAAC;QACH,CAAC;QACD,aAAa,CAAC,GAAa;YACzB,UAAU,GAAG,GAAG,CAAC;QACnB,CAAC;QACD,IAAI,cAAc;YAChB,OAAO,cAAc,CAAC;QACxB,CAAC;KACF,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,WAAmB;IACpD,OAAO,MAAM,QAAQ,CACnB,kDAAkD,EAClD,EAAE,EAAE,EAAE,WAAW,EAAE,EACnB,yBAAyB,CAC1B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,WAAmB;IACxD,OAAO,MAAM,SAAS,CACpB,0GAA0G,EAC1G,EAAE,oBAAoB,EAAE,WAAW,EAAE,EACrC,4BAA4B,CAC7B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,+BAA+B,CAAC,WAAmB;IAChE,MAAM,UAAU,CACd,+JAA+J,EAC/J,EAAE,oBAAoB,EAAE,WAAW,EAAE,CACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,0BAA0B;IACvC,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;IACzD,MAAM,UAAU,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC;QAC7C,OAAO,EAAE,MAAM;QACf,QAAQ,EAAE,wCAAwC;QAClD,SAAS,EAAE,gBAAgB;QAC3B,UAAU,EAAE,UAAU,CAAC,SAAS;QAChC,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,SAAS,EAAE,UAAU,CAAC,GAAG;QACzB,MAAM,EAAE,SAAS;KAClB,CAAC,CAAC;IACH,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACtE,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QACzC,MAAM,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/D,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,CAAC,sBAAsB,CAAC,EAAE,yBAAyB,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,iBAAiB,CAAC,aAAa,EAAE,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,KAAK,IAAI,EAAE;QAClB,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;IACzC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAChF,MAAM,QAAQ,CAAC,GAAG,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC;QAEtC,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEzB,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE/C,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;QAChD,MAAM,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAErD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACpC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,KAAK,IAAI,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC;QAC/D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QAEvD,MAAM,cAAc,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC/D,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;QACzC,IAAI,SAAS,GAAG,MAAM,0BAA0B,EAAE,CAAC;QAEnD,MAAM,uBAAuB,GAAG,sBAAsB,EAAE,CAAC;QACzD,uBAAuB,CAAC,aAAa,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACnD,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QAC9E,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QAEnB,MAAM,IAAI,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QACxE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC1B,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;QACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC;QACvD,UAAU,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;YACzB,MAAM,OAAO,GAAG,GAAG,CAAC,IAAW,CAAC;YAChC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACzB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/B,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;YACjF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC1C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,mBAAmB,CAAC,CAAC;YACzD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;YACjE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAChE,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAE/C,wDAAwD;QACxD,MAAM,+BAA+B,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;QAExE,uBAAuB,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;QACnF,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAExB,MAAM,SAAS,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAC9D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC;QAC3E,MAAM,mBAAmB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;QAClF,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;QAClE,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC/B,MAAM,CAAC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACpC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACzC,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC;QAE3D,SAAS,GAAG,MAAM,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACpD,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QAE5C,oEAAoE;QACpE,uCAAuC;QACvC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC","sourcesContent":["import { afterAll, assert, beforeAll, beforeEach, describe, it } from 'vitest';\n\nimport * as error from '@prairielearn/error';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport { makePostgresTestUtils, queryAsync, queryRow, queryRows } from '@prairielearn/postgres';\n\nimport { SCHEMA_MIGRATIONS_PATH, init } from '../index.js';\n\nimport { BatchedMigrationJobRowSchema } from './batched-migration-job.js';\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n insertBatchedMigration,\n makeBatchedMigration,\n updateBatchedMigrationStatus,\n} from './batched-migration.js';\n\nconst postgresTestUtils = makePostgresTestUtils({\n database: 'prairielearn_migrations',\n});\n\nfunction makeTestBatchMigration() {\n let executionCount = 0;\n let failingIds: bigint[] = [];\n\n return makeBatchedMigration({\n async getParameters() {\n return {\n min: 1n,\n max: 10000n,\n batchSize: 1000,\n };\n },\n async execute(start: bigint, end: bigint) {\n executionCount += 1;\n const shouldFail = failingIds.some((id) => id >= start && id <= end);\n if (shouldFail) {\n // Throw an error with some data to make sure it gets persisted. We\n // specifically use BigInt values here to make sure that they are\n // correctly serialized to strings.\n throw new error.AugmentedError('Execution failure', { data: { start, end } });\n }\n },\n setFailingIds(ids: bigint[]) {\n failingIds = ids;\n },\n get executionCount() {\n return executionCount;\n },\n });\n}\n\nasync function getBatchedMigration(migrationId: string) {\n return await queryRow(\n 'SELECT * FROM batched_migrations WHERE id = $id;',\n { id: migrationId },\n BatchedMigrationRowSchema,\n );\n}\n\nasync function getBatchedMigrationJobs(migrationId: string) {\n return await queryRows(\n 'SELECT * FROM batched_migration_jobs WHERE batched_migration_id = $batched_migration_id ORDER BY id ASC;',\n { batched_migration_id: migrationId },\n BatchedMigrationJobRowSchema,\n );\n}\n\nasync function resetFailedBatchedMigrationJobs(migrationId: string) {\n await queryAsync(\n \"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'\",\n { batched_migration_id: migrationId },\n );\n}\n\nasync function insertTestBatchedMigration() {\n const migrationImplementation = makeTestBatchMigration();\n const parameters = await migrationImplementation.getParameters();\n const migration = await insertBatchedMigration({\n project: 'test',\n filename: '20230406184103_test_batch_migration.js',\n timestamp: '20230406184103',\n batch_size: parameters.batchSize,\n min_value: parameters.min,\n max_value: parameters.max,\n status: 'running',\n });\n if (!migration) throw new Error('Failed to insert batched migration');\n return migration;\n}\n\ndescribe('BatchedMigrationExecutor', () => {\n beforeAll(async () => {\n await postgresTestUtils.createDatabase();\n await namedLocks.init(postgresTestUtils.getPoolConfig(), (err) => {\n throw err;\n });\n await init([SCHEMA_MIGRATIONS_PATH], 'prairielearn_migrations');\n });\n\n beforeEach(async () => {\n await postgresTestUtils.resetDatabase();\n });\n\n afterAll(async () => {\n await namedLocks.close();\n await postgresTestUtils.dropDatabase();\n });\n\n it('runs one iteration of a batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const executor = new BatchedMigrationRunner(migration, migrationImplementation);\n await executor.run({ iterations: 1 });\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 1);\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'running');\n\n assert.equal(migrationImplementation.executionCount, 1);\n });\n\n it('runs an entire batched migration', async () => {\n const migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n assert.lengthOf(jobs, 10);\n assert.equal(jobs[0].min_value, 1n);\n assert.equal(jobs[0].max_value, 1000n);\n assert.equal(jobs.at(-1)?.min_value, 9001n);\n assert.equal(jobs.at(-1)?.max_value, 10000n);\n assert.isTrue(jobs.every((job) => job.started_at !== null));\n assert.isTrue(jobs.every((job) => job.finished_at !== null));\n assert.isTrue(jobs.every((job) => job.status === 'succeeded'));\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n\n const finalMigration = await getBatchedMigration(migration.id);\n assert.equal(finalMigration.status, 'succeeded');\n });\n\n it('handles failing execution', async () => {\n let migration = await insertTestBatchedMigration();\n\n const migrationImplementation = makeTestBatchMigration();\n migrationImplementation.setFailingIds([1n, 5010n]);\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n await runner.run();\n\n const jobs = await getBatchedMigrationJobs(migration.id);\n const failedJobs = jobs.filter((job) => job.status === 'failed');\n const successfulJobs = jobs.filter((job) => job.status === 'succeeded');\n assert.lengthOf(jobs, 10);\n assert.lengthOf(failedJobs, 2);\n assert.lengthOf(successfulJobs, 8);\n assert.equal(migrationImplementation.executionCount, 10);\n assert.isTrue(jobs.every((job) => job.attempts === 1));\n failedJobs.forEach((job) => {\n const jobData = job.data as any;\n assert.isObject(jobData);\n assert.isObject(jobData.error);\n assert.hasAllKeys(jobData.error, ['name', 'message', 'stack', 'data', 'status']);\n assert.equal(jobData.error.name, 'Error');\n assert.equal(jobData.error.message, 'Execution failure');\n assert.equal(jobData.error.data.start, job.min_value.toString());\n assert.equal(jobData.error.data.end, job.max_value.toString());\n });\n\n const failedMigration = await getBatchedMigration(migration.id);\n assert.equal(failedMigration.status, 'failed');\n\n // Retry the failed jobs; ensure they succeed this time.\n await resetFailedBatchedMigrationJobs(migration.id);\n migration = await updateBatchedMigrationStatus(migration.id, 'running');\n\n migrationImplementation.setFailingIds([]);\n const retryRunner = new BatchedMigrationRunner(migration, migrationImplementation);\n await retryRunner.run();\n\n const finalJobs = await getBatchedMigrationJobs(migration.id);\n const finalFailedJobs = finalJobs.filter((job) => job.status === 'failed');\n const finalSuccessfulJobs = finalJobs.filter((job) => job.status === 'succeeded');\n const retriedJobs = finalJobs.filter((job) => job.attempts === 2);\n assert.lengthOf(finalJobs, 10);\n assert.lengthOf(finalFailedJobs, 0);\n assert.lengthOf(finalSuccessfulJobs, 10);\n assert.lengthOf(retriedJobs, 2);\n assert.isTrue(finalJobs.every((job) => job.data === null));\n\n migration = await getBatchedMigration(migration.id);\n assert.equal(migration.status, 'succeeded');\n\n // The runner should have run only the previously failed jobs, which\n // works out to 2 additional execution.\n assert.equal(migrationImplementation.executionCount, 12);\n });\n});\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prairielearn/migrations",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.22",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -14,22 +14,22 @@
|
|
|
14
14
|
"test": "vitest run --coverage"
|
|
15
15
|
},
|
|
16
16
|
"dependencies": {
|
|
17
|
-
"@prairielearn/error": "^2.0.
|
|
18
|
-
"@prairielearn/logger": "^2.0.
|
|
19
|
-
"@prairielearn/named-locks": "^3.0.
|
|
20
|
-
"@prairielearn/postgres": "^
|
|
17
|
+
"@prairielearn/error": "^2.0.18",
|
|
18
|
+
"@prairielearn/logger": "^2.0.19",
|
|
19
|
+
"@prairielearn/named-locks": "^3.0.22",
|
|
20
|
+
"@prairielearn/postgres": "^4.0.1",
|
|
21
21
|
"fs-extra": "^11.3.0",
|
|
22
22
|
"serialize-error": "^12.0.0",
|
|
23
|
-
"zod": "^3.25.
|
|
23
|
+
"zod": "^3.25.76"
|
|
24
24
|
},
|
|
25
25
|
"devDependencies": {
|
|
26
26
|
"@prairielearn/tsconfig": "^0.0.0",
|
|
27
27
|
"@types/fs-extra": "^11.0.4",
|
|
28
|
-
"@types/node": "^22.
|
|
28
|
+
"@types/node": "^22.17.0",
|
|
29
29
|
"@vitest/coverage-v8": "^3.2.4",
|
|
30
30
|
"tmp-promise": "^3.0.3",
|
|
31
31
|
"tsx": "^4.20.3",
|
|
32
|
-
"typescript": "^5.
|
|
32
|
+
"typescript": "^5.9.2",
|
|
33
33
|
"typescript-cp": "^0.1.9",
|
|
34
34
|
"vitest": "^3.2.4"
|
|
35
35
|
}
|
|
@@ -69,9 +69,7 @@ async function getBatchedMigrationJobs(migrationId: string) {
|
|
|
69
69
|
async function resetFailedBatchedMigrationJobs(migrationId: string) {
|
|
70
70
|
await queryAsync(
|
|
71
71
|
"UPDATE batched_migration_jobs SET status = 'pending', updated_at = CURRENT_TIMESTAMP WHERE batched_migration_id = $batched_migration_id AND status = 'failed'",
|
|
72
|
-
{
|
|
73
|
-
batched_migration_id: migrationId,
|
|
74
|
-
},
|
|
72
|
+
{ batched_migration_id: migrationId },
|
|
75
73
|
);
|
|
76
74
|
}
|
|
77
75
|
|
|
@@ -65,9 +65,7 @@ export class BatchedMigrationRunner {
|
|
|
65
65
|
private async refreshMigrationStatus(migration: BatchedMigrationRow) {
|
|
66
66
|
this.migrationStatus = await queryRow(
|
|
67
67
|
sql.get_migration_status,
|
|
68
|
-
{
|
|
69
|
-
id: migration.id,
|
|
70
|
-
},
|
|
68
|
+
{ id: migration.id },
|
|
71
69
|
BatchedMigrationStatusSchema,
|
|
72
70
|
);
|
|
73
71
|
}
|