@prairielearn/migrations 3.0.7 → 3.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. package/CHANGELOG.md +22 -0
  2. package/dist/batched-migrations/batched-migration-runner.d.ts +1 -1
  3. package/dist/batched-migrations/batched-migration-runner.js.map +1 -1
  4. package/dist/batched-migrations/batched-migrations-runner.js +2 -2
  5. package/dist/batched-migrations/batched-migrations-runner.js.map +1 -1
  6. package/dist/migrations/migrations.d.ts +1 -1
  7. package/dist/migrations/migrations.js.map +1 -1
  8. package/package.json +12 -12
  9. package/src/batched-migrations/batched-migration-runner.ts +5 -5
  10. package/src/batched-migrations/batched-migrations-runner.ts +8 -5
  11. package/src/migrations/migrations.ts +1 -1
package/CHANGELOG.md CHANGED
@@ -1,5 +1,27 @@
1
1
  # @prairielearn/migrations
2
2
 
3
+ ## 3.0.9
4
+
5
+ ### Patch Changes
6
+
7
+ - 315d931: Upgrade all JavaScript dependencies
8
+ - Updated dependencies [315d931]
9
+ - @prairielearn/named-locks@3.0.9
10
+ - @prairielearn/postgres@2.1.5
11
+ - @prairielearn/logger@2.0.7
12
+ - @prairielearn/error@2.0.7
13
+
14
+ ## 3.0.8
15
+
16
+ ### Patch Changes
17
+
18
+ - 4b79275: Upgrade all JavaScript dependencies
19
+ - Updated dependencies [4b79275]
20
+ - @prairielearn/named-locks@3.0.8
21
+ - @prairielearn/postgres@2.1.4
22
+ - @prairielearn/logger@2.0.6
23
+ - @prairielearn/error@2.0.6
24
+
3
25
  ## 3.0.7
4
26
 
5
27
  ### Patch Changes
package/dist/batched-migrations/batched-migration-runner.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { BatchedMigrationRow, BatchedMigrationImplementation } from './batched-migration.js';
1
+ import { type BatchedMigrationRow, type BatchedMigrationImplementation } from './batched-migration.js';
2
2
  interface BatchedMigrationRunnerOptions {
3
3
  logProgress?: boolean;
4
4
  }
package/dist/batched-migrations/batched-migration-runner.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,4BAA4B,GAG7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAGL,4BAA4B,EAC5B,4BAA4B,GAE7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB;YACE,EAAE,EAAE,SAAS,CAAC,EAAE;SACjB,EACD,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryRow, queryOptionalRow } from '@prairielearn/postgres';\n\nimport {\n BatchedMigrationJobRowSchema,\n BatchedMigrationJobStatus,\n BatchedMigrationJobRow,\n} from './batched-migration-job.js';\nimport {\n BatchedMigrationStatus,\n BatchedMigrationRow,\n updateBatchedMigrationStatus,\n BatchedMigrationStatusSchema,\n BatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n private options: BatchedMigrationRunnerOptions;\n private migration: BatchedMigrationRow;\n private migrationImplementation: BatchedMigrationImplementation;\n private migrationStatus: BatchedMigrationStatus;\n\n constructor(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n options: BatchedMigrationRunnerOptions = {},\n ) {\n this.options = options;\n this.migration = migration;\n this.migrationImplementation = migrationImplementation;\n this.migrationStatus = migration.status;\n }\n\n private log(message: string, ...meta: any[]) {\n if (this.options.logProgress) {\n logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n }\n }\n\n private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_incomplete_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_failed_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n this.migrationStatus = await queryRow(\n sql.get_migration_status,\n {\n id: migration.id,\n },\n BatchedMigrationStatusSchema,\n );\n }\n\n private async finishRunningMigration(migration: BatchedMigrationRow) {\n // Safety check: if there are any pending jobs, don't mark this\n // migration as finished.\n if (await this.hasIncompleteJobs(migration)) {\n this.log('Incomplete jobs found, not marking as finished');\n return;\n }\n\n const hasFailedJobs = await this.hasFailedJobs(migration);\n const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n await updateBatchedMigrationStatus(migration.id, finalStatus);\n this.log(`Finished with status '${finalStatus}'`);\n }\n\n private async getNextBatchBounds(\n migration: BatchedMigrationRow,\n ): Promise<null | [bigint, bigint]> {\n const lastJob = await queryOptionalRow(\n sql.select_last_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n\n const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n if (nextMin > migration.max_value) return null;\n\n let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n return [nextMin, nextMax];\n }\n\n private async startJob(job: BatchedMigrationJobRow) {\n await queryAsync(sql.start_batched_migration_job, { id: job.id });\n const jobRange = `[${job.min_value}, ${job.max_value}]`;\n const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n }\n\n private serializeJobData(data: unknown) {\n if (data == null) return null;\n\n // Return JSON-stringified data. Convert BigInts to strings.\n return JSON.stringify(data, (_key, value) => {\n if (typeof value === 'bigint') return value.toString();\n return value;\n });\n }\n\n private async finishJob(\n job: BatchedMigrationJobRow,\n status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n data?: unknown,\n ) {\n await queryAsync(sql.finish_batched_migration_job, {\n id: job.id,\n status,\n data: this.serializeJobData(data),\n });\n this.log(`Job ${job.id} finished with status '${status}'`);\n }\n\n private async getOrCreateNextMigrationJob(\n migration: BatchedMigrationRow,\n ): Promise<BatchedMigrationJobRow | null> {\n const nextBatchBounds = await this.getNextBatchBounds(migration);\n if (nextBatchBounds) {\n return await queryRow(\n sql.insert_batched_migration_job,\n {\n batched_migration_id: migration.id,\n min_value: nextBatchBounds[0],\n max_value: nextBatchBounds[1],\n },\n BatchedMigrationJobRowSchema,\n );\n } else {\n // Pick up any old pending jobs from this migration. These will only exist if\n // an admin manually elected to retry all failed jobs; we'll never automatically\n // transition failed jobs back to pending.\n return await queryOptionalRow(\n sql.select_first_pending_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n }\n }\n\n private async runMigrationJob(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n ) {\n const nextJob = await this.getOrCreateNextMigrationJob(migration);\n if (nextJob) {\n await this.startJob(nextJob);\n\n let error = null;\n try {\n // We'll only handle errors thrown by the migration itself. If any of\n // our own execution machinery throws an error, we'll let it bubble up.\n await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n } catch (err) {\n error = err;\n }\n\n if (error) {\n logger.error(\n `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n error,\n );\n await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n } else {\n await this.finishJob(nextJob, 'succeeded');\n }\n } else {\n await this.finishRunningMigration(migration);\n }\n }\n\n async run({\n signal,\n iterations,\n durationMs,\n }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n let iterationCount = 0;\n const endTime = durationMs ? Date.now() + durationMs : null;\n while (\n !signal?.aborted &&\n (iterations ? iterationCount < iterations : true) &&\n (endTime ? Date.now() < endTime : true) &&\n (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n ) {\n await this.runMigrationJob(this.migration, this.migrationImplementation);\n iterationCount += 1;\n // Always refresh the status so we can detect if the migration was marked\n // as paused by another process.\n await this.refreshMigrationStatus(this.migration);\n }\n }\n}\n"]}
1
+ {"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,4BAA4B,GAG7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAGL,4BAA4B,EAC5B,4BAA4B,GAE7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB;YACE,EAAE,EAAE,SAAS,CAAC,EAAE;SACjB,EACD,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryRow, queryOptionalRow } from '@prairielearn/postgres';\n\nimport {\n BatchedMigrationJobRowSchema,\n type BatchedMigrationJobStatus,\n type BatchedMigrationJobRow,\n} from './batched-migration-job.js';\nimport {\n type BatchedMigrationStatus,\n type BatchedMigrationRow,\n updateBatchedMigrationStatus,\n BatchedMigrationStatusSchema,\n type BatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n private options: BatchedMigrationRunnerOptions;\n private migration: BatchedMigrationRow;\n private migrationImplementation: BatchedMigrationImplementation;\n private migrationStatus: BatchedMigrationStatus;\n\n constructor(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n options: BatchedMigrationRunnerOptions = {},\n ) {\n this.options = options;\n this.migration = migration;\n this.migrationImplementation = migrationImplementation;\n this.migrationStatus = migration.status;\n }\n\n private log(message: string, ...meta: any[]) {\n if (this.options.logProgress) {\n logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n }\n }\n\n private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_incomplete_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n return await queryRow(\n sql.batched_migration_has_failed_jobs,\n { batched_migration_id: migration.id },\n z.boolean(),\n );\n }\n\n private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n this.migrationStatus = await queryRow(\n sql.get_migration_status,\n {\n id: migration.id,\n },\n BatchedMigrationStatusSchema,\n );\n }\n\n private async finishRunningMigration(migration: BatchedMigrationRow) {\n // Safety check: if there are any pending jobs, don't mark this\n // migration as finished.\n if (await this.hasIncompleteJobs(migration)) {\n this.log('Incomplete jobs found, not marking as finished');\n return;\n }\n\n const hasFailedJobs = await this.hasFailedJobs(migration);\n const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n await updateBatchedMigrationStatus(migration.id, finalStatus);\n this.log(`Finished with status '${finalStatus}'`);\n }\n\n private async getNextBatchBounds(\n migration: BatchedMigrationRow,\n ): Promise<null | [bigint, bigint]> {\n const lastJob = await queryOptionalRow(\n sql.select_last_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n\n const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n if (nextMin > migration.max_value) return null;\n\n let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n return [nextMin, nextMax];\n }\n\n private async startJob(job: BatchedMigrationJobRow) {\n await queryAsync(sql.start_batched_migration_job, { id: job.id });\n const jobRange = `[${job.min_value}, ${job.max_value}]`;\n const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n }\n\n private serializeJobData(data: unknown) {\n if (data == null) return null;\n\n // Return JSON-stringified data. Convert BigInts to strings.\n return JSON.stringify(data, (_key, value) => {\n if (typeof value === 'bigint') return value.toString();\n return value;\n });\n }\n\n private async finishJob(\n job: BatchedMigrationJobRow,\n status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n data?: unknown,\n ) {\n await queryAsync(sql.finish_batched_migration_job, {\n id: job.id,\n status,\n data: this.serializeJobData(data),\n });\n this.log(`Job ${job.id} finished with status '${status}'`);\n }\n\n private async getOrCreateNextMigrationJob(\n migration: BatchedMigrationRow,\n ): Promise<BatchedMigrationJobRow | null> {\n const nextBatchBounds = await this.getNextBatchBounds(migration);\n if (nextBatchBounds) {\n return await queryRow(\n sql.insert_batched_migration_job,\n {\n batched_migration_id: migration.id,\n min_value: nextBatchBounds[0],\n max_value: nextBatchBounds[1],\n },\n BatchedMigrationJobRowSchema,\n );\n } else {\n // Pick up any old pending jobs from this migration. These will only exist if\n // an admin manually elected to retry all failed jobs; we'll never automatically\n // transition failed jobs back to pending.\n return await queryOptionalRow(\n sql.select_first_pending_batched_migration_job,\n { batched_migration_id: migration.id },\n BatchedMigrationJobRowSchema,\n );\n }\n }\n\n private async runMigrationJob(\n migration: BatchedMigrationRow,\n migrationImplementation: BatchedMigrationImplementation,\n ) {\n const nextJob = await this.getOrCreateNextMigrationJob(migration);\n if (nextJob) {\n await this.startJob(nextJob);\n\n let error = null;\n try {\n // We'll only handle errors thrown by the migration itself. If any of\n // our own execution machinery throws an error, we'll let it bubble up.\n await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n } catch (err) {\n error = err;\n }\n\n if (error) {\n logger.error(\n `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n error,\n );\n await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n } else {\n await this.finishJob(nextJob, 'succeeded');\n }\n } else {\n await this.finishRunningMigration(migration);\n }\n }\n\n async run({\n signal,\n iterations,\n durationMs,\n }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n let iterationCount = 0;\n const endTime = durationMs ? Date.now() + durationMs : null;\n while (\n !signal?.aborted &&\n (iterations ? iterationCount < iterations : true) &&\n (endTime ? Date.now() < endTime : true) &&\n (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n ) {\n await this.runMigrationJob(this.migration, this.migrationImplementation);\n iterationCount += 1;\n // Always refresh the status so we can detect if the migration was marked\n // as paused by another process.\n await this.refreshMigrationStatus(this.migration);\n }\n }\n}\n"]}
package/dist/batched-migrations/batched-migrations-runner.js CHANGED
@@ -3,7 +3,7 @@ import path from 'node:path';
3
3
  import { setTimeout as sleep } from 'node:timers/promises';
4
4
  import { doWithLock } from '@prairielearn/named-locks';
5
5
  import { loadSqlEquiv, queryOptionalRow } from '@prairielearn/postgres';
6
- import { readAndValidateMigrationsFromDirectories } from '../load-migrations.js';
6
+ import { readAndValidateMigrationsFromDirectories, } from '../load-migrations.js';
7
7
  import { BatchedMigrationRunner } from './batched-migration-runner.js';
8
8
  import { BatchedMigrationRowSchema, insertBatchedMigration, selectBatchedMigrationForTimestamp, updateBatchedMigrationStatus, validateBatchedMigrationImplementation, } from './batched-migration.js';
9
9
  const sql = loadSqlEquiv(import.meta.filename);
@@ -136,7 +136,7 @@ export class BatchedMigrationsRunner extends EventEmitter {
136
136
  try {
137
137
  await sleep(sleepDurationMs, null, { ref: false, signal: this.abortController.signal });
138
138
  }
139
- catch (err) {
139
+ catch {
140
140
  // We don't care about errors here, they should only ever occur when
141
141
  // the AbortController is aborted. Continue to the next iteration of
142
142
  // the loop so we can shut down.
package/dist/batched-migrations/batched-migrations-runner.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"batched-migrations-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migrations-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,EAAiB,wCAAwC,EAAE,MAAM,uBAAuB,CAAC;AAEhG,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EAEzB,sBAAsB,EAEtB,kCAAkC,EAClC,4BAA4B,EAE5B,sCAAsC,GACvC,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAE/C,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAgBlD,MAAM,OAAO,uBAAwB,SAAQ,YAAY;IACtC,OAAO,CAAgC;IACvC,QAAQ,CAAS;IAC1B,OAAO,GAAG,KAAK,CAAC;IAChB,cAAc,GAA2B,IAAI,CAAC;IAC9C,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAEhD,YAAY,OAAsC;QAChD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,sBAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/D,CAAC;IAEO,oBAAoB,CAAC,SAAiB;QAC5C,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;IACzC,CAAC;IAEO,iBAAiB,GAAG,KAAK,IAAI,EAAE;QACrC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,MAAM,wCAAwC,CAClE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,UAAU,CACX,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC,CAAC;IAEM,KAAK,CAAC,yBAAyB,CAAC,UAAkB;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;QAC5E,OAAO,aAAa,IAAI,IAAI,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,2BAA2B,CAAC,aAA4B;QACpE,6DAA6D;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QACvF,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAE1D,MAAM,uBAAuB,GAAG,eAAe,CAAC,OAAyC,CAAC;QAC1F,sCAAsC,CAAC,uBAAuB,CAAC,CAAC;QAChE,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,UAAkB;QAC9C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACvE,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;QACtF,MAAM,mBAAmB,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;QAE1E,uEAAuE;QACvE,qDAAqD;QACrD,MAAM,MAAM,GACV,mBAAmB,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAEtE,MAAM,sBAAsB,CAAC;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YAC7B,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,SAAS,EAAE,aAAa,CAAC,SAAS;YAClC,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,QAAQ;YACnB,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,UAAkB,EAAE,OAAyC;QAC1F,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,IAAI,SAAS,GAAG,MAAM,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAE1F,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QAE7C,2EAA2E;QAC3E,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,UAAU,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;YACrF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;YACvE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;YAEtF,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,EAAE;gBAC5E,kDAAkD;gBAClD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,IAAI;aAC1C,CAAC,CAAC;YAEH,uEAAuE;YACvE,2BAA2B;YAC3B,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,SAAS,GAAG,MAAM,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QAE7C,MAAM,IAAI,KAAK,CACb,8CAA8C,UAAU,4CAA4C,SAAS,CAAC,MAAM,IAAI,CACzH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAwC,EAAE;QAC9C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,eAAe,EAAgC;QAC1E,cAAc,KAAK,wBAAwB,CAAC;QAC5C,eAAe,KAAK,yBAAyB,CAAC;QAE9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACxC,uEAAuE;gBACvE,mBAAmB;gBACnB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,OAAO;YACT,CAAC;YAED,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACxD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;YAED,0EAA0E;YAC1E,0EAA0E;YAC1E,yBAAyB;YACzB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,qEAAqE;gBACrE,4BAA4B;gBAC5B,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC1F,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,oEAAoE;oBACpE,oEAAoE;oBACpE,gCAAgC;oBAChC,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,UAAU,CACf,IAAI,CAAC,QAAQ,EACb;YACE,2DAA2D;YAC3D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,EACD,KAAK,IAAI,EAAE;YACT,IAAI,SAAS,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,wBAAwB,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACjC,yBAAyB,CAC1B,CAAC;YAEF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,SAAS,GAAG,MAAM,gBAAgB,CAChC,GAAG,CAAC,4BAA4B,EAChC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACjC,yBAAyB,CAC1B,CAAC;YACJ,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACvC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,mCAAmC;YACnC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,uEAAuE;QACvE,+CAA+C;QAC/C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAChF,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,UAAU,CACd,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,SAAS,CAAC,EAClD;YACE,SAAS,EAAE,IAAI;YACf,4DAA4D;YAC5D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,EACD,KAAK,IAAI,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;YAEtF,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YAE9E,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sCAAsC;QACtC,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;CACF;AAED,IAAI,MAAM,GAAmC,IAAI,CAAC;AAElD,SAAS,YAAY,CACnB,MAAsC;IAEtC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAsC;IAC1E,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACtE,MAAM,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAwC,EAAE;IAC/E,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACpB,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,UAAkB;IAC9D,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC;AACnD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,UAAkB,EAClB,OAAyC;IAEzC,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC7D,CAAC","sourcesContent":["import { EventEmitter } from 'node:events';\nimport path from 'node:path';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nimport { doWithLock } from '@prairielearn/named-locks';\nimport { loadSqlEquiv, queryOptionalRow } from '@prairielearn/postgres';\n\nimport { MigrationFile, readAndValidateMigrationsFromDirectories } from '../load-migrations.js';\n\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n BatchedMigrationRow,\n insertBatchedMigration,\n BatchedMigrationStatus,\n selectBatchedMigrationForTimestamp,\n updateBatchedMigrationStatus,\n BatchedMigrationImplementation,\n validateBatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\nconst DEFAULT_MIN_VALUE = 1n;\nconst DEFAULT_BATCH_SIZE = 1_000;\nconst DEFAULT_WORK_DURATION_MS = 60_000;\nconst DEFAULT_SLEEP_DURATION_MS = 30_000;\nconst EXTENSIONS = ['.js', '.ts', '.mjs', '.mts'];\n\ninterface BatchedMigrationRunnerOptions {\n project: string;\n directories: string[];\n}\n\ninterface BatchedMigrationStartOptions {\n workDurationMs?: number;\n sleepDurationMs?: number;\n}\n\ninterface BatchedMigrationFinalizeOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationsRunner extends EventEmitter {\n private readonly options: BatchedMigrationRunnerOptions;\n private readonly lockName: string;\n private running = false;\n private migrationFiles: MigrationFile[] | null = null;\n private abortController = new AbortController();\n\n constructor(options: BatchedMigrationRunnerOptions) {\n super();\n this.options = options;\n this.lockName = `batched-migrations:${this.options.project}`;\n }\n\n private lockNameForTimestamp(timestamp: string) {\n return `${this.lockName}:${timestamp}`;\n }\n\n private getMigrationFiles = async () => {\n if (!this.migrationFiles) {\n this.migrationFiles = await readAndValidateMigrationsFromDirectories(\n this.options.directories,\n EXTENSIONS,\n );\n }\n return this.migrationFiles;\n };\n\n private async getMigrationForIdentifier(identifier: string): Promise<MigrationFile | null> {\n const timestamp = identifier.split('_')[0];\n\n const migrationFiles = await this.getMigrationFiles();\n const migrationFile = migrationFiles.find((m) => m.timestamp === timestamp);\n return migrationFile ?? null;\n }\n\n /**\n * Loads the implementation for the migration with the given identifier. The identifier\n * must start with a 14-character timestamp. It may optionally be followed by\n * an underscore with additional characters, which are ignored. These should\n * typically be used to provide a human-readable name for the migration.\n */\n private async loadMigrationImplementation(migrationFile: MigrationFile) {\n // We use dynamic imports to handle both CJS and ESM modules.\n const migrationModulePath = path.join(migrationFile.directory, migrationFile.filename);\n const migrationModule = await import(migrationModulePath);\n\n const migrationImplementation = migrationModule.default as BatchedMigrationImplementation;\n validateBatchedMigrationImplementation(migrationImplementation);\n return migrationImplementation;\n }\n\n async enqueueBatchedMigration(identifier: string) {\n const migrationFile = await this.getMigrationForIdentifier(identifier);\n if (!migrationFile) {\n throw new Error(`No migration found for identifier ${identifier}`);\n }\n\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n const migrationParameters = await migrationImplementation.getParameters();\n\n // If `max` is null, that implies that there are no rows to process, so\n // we can immediately mark the migration as finished.\n const status: BatchedMigrationStatus =\n migrationParameters.max === null ? 'succeeded' : 'pending';\n\n const minValue = BigInt(migrationParameters.min ?? DEFAULT_MIN_VALUE);\n const maxValue = BigInt(migrationParameters.max ?? minValue);\n const batchSize = migrationParameters.batchSize ?? DEFAULT_BATCH_SIZE;\n\n await insertBatchedMigration({\n project: this.options.project,\n filename: migrationFile.filename,\n timestamp: migrationFile.timestamp,\n batch_size: batchSize,\n min_value: minValue,\n max_value: maxValue,\n status,\n });\n }\n\n async finalizeBatchedMigration(identifier: string, options?: BatchedMigrationFinalizeOptions) {\n const timestamp = identifier.split('_')[0];\n\n let migration = await selectBatchedMigrationForTimestamp(this.options.project, timestamp);\n\n if (migration.status === 'succeeded') return;\n\n // If the migration isn't already in the finalizing state, mark it as such.\n if (migration.status !== 'finalizing') {\n migration = await updateBatchedMigrationStatus(migration.id, 'finalizing');\n }\n\n await doWithLock(this.lockNameForTimestamp(timestamp), { autoRenew: true }, async () => {\n const migrationFile = await this.getMigrationForIdentifier(identifier);\n if (!migrationFile) {\n throw new Error(`No migration found for identifier ${identifier}`);\n }\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n\n const runner = new BatchedMigrationRunner(migration, migrationImplementation, {\n // Always log progress unless explicitly disabled.\n logProgress: options?.logProgress ?? true,\n });\n\n // Because we don't give any arguments to `run()`, it will run until it\n // has attempted every job.\n await runner.run();\n });\n\n migration = await selectBatchedMigrationForTimestamp(this.options.project, timestamp);\n\n if (migration.status === 'succeeded') return;\n\n throw new Error(\n `Expected batched migration with identifier ${identifier} to be marked as 'succeeded', but it is '${migration.status}'.`,\n );\n }\n\n start(options: BatchedMigrationStartOptions = {}) {\n if (this.running) {\n throw new Error('BatchedMigrationsRunner is already running');\n }\n\n this.loop(options);\n }\n\n async loop({ workDurationMs, sleepDurationMs }: BatchedMigrationStartOptions) {\n workDurationMs ??= DEFAULT_WORK_DURATION_MS;\n sleepDurationMs ??= DEFAULT_SLEEP_DURATION_MS;\n\n this.running = true;\n while (this.running) {\n if (this.abortController.signal.aborted) {\n // We assign this here so that `stop()` can tell when this loop is done\n // processing jobs.\n this.running = false;\n return;\n }\n\n let didWork = false;\n try {\n didWork = await this.maybePerformWork(workDurationMs);\n } catch (err) {\n this.emit('error', err);\n }\n\n // If we did work, we'll immediately try again since there's probably more\n // work to be done. If not, we'll sleep for a while - maybe some more work\n // will become available!\n if (!didWork) {\n // We provide the signal here so that we can more quickly stop things\n // when we're shutting down.\n try {\n await sleep(sleepDurationMs, null, { ref: false, signal: this.abortController.signal });\n } catch (err) {\n // We don't care about errors here, they should only ever occur when\n // the AbortController is aborted. Continue to the next iteration of\n // the loop so we can shut down.\n continue;\n }\n }\n }\n }\n\n private async getOrStartMigration(): Promise<BatchedMigrationRow | null> {\n return doWithLock(\n this.lockName,\n {\n // Don't fail if the lock couldn't be acquired immediately.\n onNotAcquired: () => null,\n },\n async () => {\n let migration = await queryOptionalRow(\n sql.select_running_migration,\n { project: this.options.project },\n BatchedMigrationRowSchema,\n );\n\n if (!migration) {\n migration = await queryOptionalRow(\n sql.start_next_pending_migration,\n { project: this.options.project },\n BatchedMigrationRowSchema,\n );\n }\n\n return migration;\n },\n );\n }\n\n async maybePerformWork(durationMs: number): Promise<boolean> {\n const migration = await this.getOrStartMigration();\n if (!migration) {\n // No work to do. Handle this case.\n return false;\n }\n\n // This server may not yet know about the current running migration. If\n // that's the case, we'll just skip it for now.\n const migrationFile = await this.getMigrationForIdentifier(migration.timestamp);\n if (!migrationFile) {\n return false;\n }\n\n let didWork = false;\n await doWithLock(\n this.lockNameForTimestamp(migrationFile.timestamp),\n {\n autoRenew: true,\n // Do nothing if the lock could not immediately be acquired.\n onNotAcquired: () => null,\n },\n async () => {\n didWork = true;\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n\n try {\n await runner.run({ signal: this.abortController.signal, durationMs });\n } catch (err) {\n this.emit('error', err);\n }\n },\n );\n\n return didWork;\n }\n\n async stop() {\n this.abortController.abort();\n\n // Spin until we're no longer running.\n while (this.running) {\n await sleep(1000);\n }\n }\n}\n\nlet runner: BatchedMigrationsRunner | null = null;\n\nfunction assertRunner(\n runner: BatchedMigrationsRunner | null,\n): asserts runner is BatchedMigrationsRunner {\n if (!runner) throw new Error('Batched migrations not initialized');\n}\n\nexport function initBatchedMigrations(options: BatchedMigrationRunnerOptions) {\n if (runner) throw new Error('Batched migrations already initialized');\n runner = new BatchedMigrationsRunner(options);\n return runner;\n}\n\nexport function startBatchedMigrations(options: BatchedMigrationStartOptions = {}) {\n assertRunner(runner);\n runner.start(options);\n return runner;\n}\n\nexport async function stopBatchedMigrations() {\n assertRunner(runner);\n await runner.stop();\n runner = null;\n}\n\n/**\n * Given a batched migration identifier like `20230406184103_migration`,\n * enqueues it for execution by creating a row in the `batched_migrations`\n * table.\n *\n * Despite taking a full identifier, only the timestamp is used to uniquely\n * identify the batched migration. The remaining part is just used to make\n * calls more human-readable.\n *\n * @param identifier The identifier of the batched migration to enqueue.\n */\nexport async function enqueueBatchedMigration(identifier: string) {\n assertRunner(runner);\n await runner.enqueueBatchedMigration(identifier);\n}\n\n/**\n * Given a batched migration identifier like `20230406184103_migration`,\n * synchronously runs it to completion. An error will be thrown if the final\n * status of the migration is not `succeeded`.\n *\n * @param identifier The identifier of the batched migration to finalize.\n * @param options Options for finalizing the batched migration.\n */\nexport async function finalizeBatchedMigration(\n identifier: string,\n options?: BatchedMigrationFinalizeOptions,\n) {\n assertRunner(runner);\n await runner.finalizeBatchedMigration(identifier, options);\n}\n"]}
1
+ {"version":3,"file":"batched-migrations-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migrations-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAExE,OAAO,EAEL,wCAAwC,GACzC,MAAM,uBAAuB,CAAC;AAE/B,OAAO,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACvE,OAAO,EACL,yBAAyB,EAEzB,sBAAsB,EAEtB,kCAAkC,EAClC,4BAA4B,EAE5B,sCAAsC,GACvC,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAE/C,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,wBAAwB,GAAG,MAAM,CAAC;AACxC,MAAM,yBAAyB,GAAG,MAAM,CAAC;AACzC,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAgBlD,MAAM,OAAO,uBAAwB,SAAQ,YAAY;IACtC,OAAO,CAAgC;IACvC,QAAQ,CAAS;IAC1B,OAAO,GAAG,KAAK,CAAC;IAChB,cAAc,GAA2B,IAAI,CAAC;IAC9C,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;IAEhD,YAAY,OAAsC;QAChD,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,sBAAsB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/D,CAAC;IAEO,oBAAoB,CAAC,SAAiB;QAC5C,OAAO,GAAG,IAAI,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;IACzC,CAAC;IAEO,iBAAiB,GAAG,KAAK,IAAI,EAAE;QACrC,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,IAAI,CAAC,cAAc,GAAG,MAAM,wCAAwC,CAClE,IAAI,CAAC,OAAO,CAAC,WAAW,EACxB,UAAU,CACX,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC,cAAc,CAAC;IAC7B,CAAC,CAAC;IAEM,KAAK,CAAC,yBAAyB,CAAC,UAAkB;QACxD,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACtD,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;QAC5E,OAAO,aAAa,IAAI,IAAI,CAAC;IAC/B,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,2BAA2B,CAAC,aAA4B;QACpE,6DAA6D;QAC7D,MAAM,mBAAmB,GAAG,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QACvF,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;QAE1D,MAAM,uBAAuB,GAAG,eAAe,CAAC,OAAyC,CAAC;QAC1F,sCAAsC,CAAC,uBAAuB,CAAC,CAAC;QAChE,OAAO,uBAAuB,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,UAAkB;QAC9C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;QACvE,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;QACtF,MAAM,mBAAmB,GAAG,MAAM,uBAAuB,CAAC,aAAa,EAAE,CAAC;QAE1E,uEAAuE;QACvE,qDAAqD;QACrD,MAAM,MAAM,GACV,mBAAmB,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QAE7D,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,IAAI,iBAAiB,CAAC,CAAC;QACtE,MAAM,QAAQ,GAAG,MAAM,CAAC,mBAAmB,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC;QAC7D,MAAM,SAAS,GAAG,mBAAmB,CAAC,SAAS,IAAI,kBAAkB,CAAC;QAEtE,MAAM,sBAAsB,CAAC;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;YAC7B,QAAQ,EAAE,aAAa,CAAC,QAAQ;YAChC,SAAS,EAAE,aAAa,CAAC,SAAS;YAClC,UAAU,EAAE,SAAS;YACrB,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,QAAQ;YACnB,MAAM;SACP,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,wBAAwB,CAAC,UAAkB,EAAE,OAAyC;QAC1F,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3C,IAAI,SAAS,GAAG,MAAM,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAE1F,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QAE7C,2EAA2E;QAC3E,IAAI,SAAS,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YACtC,SAAS,GAAG,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,YAAY,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,UAAU,CAAC,IAAI,CAAC,oBAAoB,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,EAAE,KAAK,IAAI,EAAE;YACrF,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,UAAU,CAAC,CAAC;YACvE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,qCAAqC,UAAU,EAAE,CAAC,CAAC;YACrE,CAAC;YACD,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;YAEtF,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,EAAE;gBAC5E,kDAAkD;gBAClD,WAAW,EAAE,OAAO,EAAE,WAAW,IAAI,IAAI;aAC1C,CAAC,CAAC;YAEH,uEAAuE;YACvE,2BAA2B;YAC3B,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;QACrB,CAAC,CAAC,CAAC;QAEH,SAAS,GAAG,MAAM,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,WAAW;YAAE,OAAO;QAE7C,MAAM,IAAI,KAAK,CACb,8CAA8C,UAAU,4CAA4C,SAAS,CAAC,MAAM,IAAI,CACzH,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAwC,EAAE;QAC9C,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,cAAc,EAAE,eAAe,EAAgC;QAC1E,cAAc,KAAK,wBAAwB,CAAC;QAC5C,eAAe,KAAK,yBAAyB,CAAC;QAE9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACxC,uEAAuE;gBACvE,mBAAmB;gBACnB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;gBACrB,OAAO;YACT,CAAC;YAED,IAAI,OAAO,GAAG,KAAK,CAAC;YACpB,IAAI,CAAC;gBACH,OAAO,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;YACxD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;YAED,0EAA0E;YAC1E,0EAA0E;YAC1E,yBAAyB;YACzB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,qEAAqE;gBACrE,4BAA4B;gBAC5B,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC1F,CAAC;gBAAC,MAAM,CAAC;oBACP,oEAAoE;oBACpE,oEAAoE;oBACpE,gCAAgC;oBAChC,SAAS;gBACX,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB;QAC/B,OAAO,UAAU,CACf,IAAI,CAAC,QAAQ,EACb;YACE,2DAA2D;YAC3D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,EACD,KAAK,IAAI,EAAE;YACT,IAAI,SAAS,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,wBAAwB,EAC5B,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACjC,yBAAyB,CAC1B,CAAC;YAEF,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,SAAS,GAAG,MAAM,gBAAgB,CAChC,GAAG,CAAC,4BAA4B,EAChC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EACjC,yBAAyB,CAC1B,CAAC;YACJ,CAAC;YAED,OAAO,SAAS,CAAC;QACnB,CAAC,CACF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACvC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACnD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,mCAAmC;YACnC,OAAO,KAAK,CAAC;QACf,CAAC;QAED,uEAAuE;QACvE,+CAA+C;QAC/C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;QAChF,IAAI,CAAC,aAAa,EAAE,CAAC;YACnB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,UAAU,CACd,IAAI,CAAC,oBAAoB,CAAC,aAAa,CAAC,SAAS,CAAC,EAClD;YACE,SAAS,EAAE,IAAI;YACf,4DAA4D;YAC5D,aAAa,EAAE,GAAG,EAAE,CAAC,IAAI;SAC1B,EACD,KAAK,IAAI,EAAE;YACT,OAAO,GAAG,IAAI,CAAC;YACf,MAAM,uBAAuB,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,aAAa,CAAC,CAAC;YAEtF,MAAM,MAAM,GAAG,IAAI,sBAAsB,CAAC,SAAS,EAAE,uBAAuB,CAAC,CAAC;YAE9E,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CACF,CAAC;QAEF,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;QAE7B,sCAAsC;QACtC,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;CACF;AAED,IAAI,MAAM,GAAmC,IAAI,CAAC;AAElD,SAAS,YAAY,CACnB,MAAsC;IAEtC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAsC;IAC1E,IAAI,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACtE,MAAM,GAAG,IAAI,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,UAAwC,EAAE;IAC/E,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;IACpB,MAAM,GAAG,IAAI,CAAC;AAChB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,UAAkB;IAC9D,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,uBAAuB,CAAC,UAAU,CAAC,CAAC;AACnD,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,UAAkB,EAClB,OAAyC;IAEzC,YAAY,CAAC,MAAM,CAAC,CAAC;IACrB,MAAM,MAAM,CAAC,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;AAC7D,CAAC","sourcesContent":["import { EventEmitter } from 'node:events';\nimport path from 'node:path';\nimport { setTimeout as sleep } from 'node:timers/promises';\n\nimport { doWithLock } from '@prairielearn/named-locks';\nimport { loadSqlEquiv, queryOptionalRow } from '@prairielearn/postgres';\n\nimport {\n type MigrationFile,\n readAndValidateMigrationsFromDirectories,\n} from '../load-migrations.js';\n\nimport { BatchedMigrationRunner } from './batched-migration-runner.js';\nimport {\n BatchedMigrationRowSchema,\n type BatchedMigrationRow,\n insertBatchedMigration,\n type BatchedMigrationStatus,\n selectBatchedMigrationForTimestamp,\n updateBatchedMigrationStatus,\n type BatchedMigrationImplementation,\n validateBatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\nconst DEFAULT_MIN_VALUE = 1n;\nconst DEFAULT_BATCH_SIZE = 1_000;\nconst DEFAULT_WORK_DURATION_MS = 60_000;\nconst DEFAULT_SLEEP_DURATION_MS = 30_000;\nconst EXTENSIONS = ['.js', '.ts', '.mjs', '.mts'];\n\ninterface BatchedMigrationRunnerOptions {\n project: string;\n directories: string[];\n}\n\ninterface BatchedMigrationStartOptions {\n workDurationMs?: number;\n sleepDurationMs?: number;\n}\n\ninterface BatchedMigrationFinalizeOptions {\n logProgress?: boolean;\n}\n\nexport class BatchedMigrationsRunner extends EventEmitter {\n private readonly options: BatchedMigrationRunnerOptions;\n private readonly lockName: string;\n private running = false;\n private migrationFiles: MigrationFile[] | null = null;\n private abortController = new AbortController();\n\n constructor(options: BatchedMigrationRunnerOptions) {\n super();\n this.options = options;\n this.lockName = `batched-migrations:${this.options.project}`;\n }\n\n private lockNameForTimestamp(timestamp: string) {\n return `${this.lockName}:${timestamp}`;\n }\n\n private getMigrationFiles = async () => {\n if (!this.migrationFiles) {\n this.migrationFiles = await readAndValidateMigrationsFromDirectories(\n this.options.directories,\n EXTENSIONS,\n );\n }\n return this.migrationFiles;\n };\n\n private async getMigrationForIdentifier(identifier: string): Promise<MigrationFile | null> {\n const timestamp = identifier.split('_')[0];\n\n const migrationFiles = await this.getMigrationFiles();\n const migrationFile = migrationFiles.find((m) => m.timestamp === timestamp);\n return migrationFile ?? null;\n }\n\n /**\n * Loads the implementation for the migration with the given identifier. The identifier\n * must start with a 14-character timestamp. It may optionally be followed by\n * an underscore with additional characters, which are ignored. These should\n * typically be used to provide a human-readable name for the migration.\n */\n private async loadMigrationImplementation(migrationFile: MigrationFile) {\n // We use dynamic imports to handle both CJS and ESM modules.\n const migrationModulePath = path.join(migrationFile.directory, migrationFile.filename);\n const migrationModule = await import(migrationModulePath);\n\n const migrationImplementation = migrationModule.default as BatchedMigrationImplementation;\n validateBatchedMigrationImplementation(migrationImplementation);\n return migrationImplementation;\n }\n\n async enqueueBatchedMigration(identifier: string) {\n const migrationFile = await this.getMigrationForIdentifier(identifier);\n if (!migrationFile) {\n throw new Error(`No migration found for identifier ${identifier}`);\n }\n\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n const migrationParameters = await migrationImplementation.getParameters();\n\n // If `max` is null, that implies that there are no rows to process, so\n // we can immediately mark the migration as finished.\n const status: BatchedMigrationStatus =\n migrationParameters.max === null ? 'succeeded' : 'pending';\n\n const minValue = BigInt(migrationParameters.min ?? DEFAULT_MIN_VALUE);\n const maxValue = BigInt(migrationParameters.max ?? minValue);\n const batchSize = migrationParameters.batchSize ?? DEFAULT_BATCH_SIZE;\n\n await insertBatchedMigration({\n project: this.options.project,\n filename: migrationFile.filename,\n timestamp: migrationFile.timestamp,\n batch_size: batchSize,\n min_value: minValue,\n max_value: maxValue,\n status,\n });\n }\n\n async finalizeBatchedMigration(identifier: string, options?: BatchedMigrationFinalizeOptions) {\n const timestamp = identifier.split('_')[0];\n\n let migration = await selectBatchedMigrationForTimestamp(this.options.project, timestamp);\n\n if (migration.status === 'succeeded') return;\n\n // If the migration isn't already in the finalizing state, mark it as such.\n if (migration.status !== 'finalizing') {\n migration = await updateBatchedMigrationStatus(migration.id, 'finalizing');\n }\n\n await doWithLock(this.lockNameForTimestamp(timestamp), { autoRenew: true }, async () => {\n const migrationFile = await this.getMigrationForIdentifier(identifier);\n if (!migrationFile) {\n throw new Error(`No migration found for identifier ${identifier}`);\n }\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n\n const runner = new BatchedMigrationRunner(migration, migrationImplementation, {\n // Always log progress unless explicitly disabled.\n logProgress: options?.logProgress ?? true,\n });\n\n // Because we don't give any arguments to `run()`, it will run until it\n // has attempted every job.\n await runner.run();\n });\n\n migration = await selectBatchedMigrationForTimestamp(this.options.project, timestamp);\n\n if (migration.status === 'succeeded') return;\n\n throw new Error(\n `Expected batched migration with identifier ${identifier} to be marked as 'succeeded', but it is '${migration.status}'.`,\n );\n }\n\n start(options: BatchedMigrationStartOptions = {}) {\n if (this.running) {\n throw new Error('BatchedMigrationsRunner is already running');\n }\n\n this.loop(options);\n }\n\n async loop({ workDurationMs, sleepDurationMs }: BatchedMigrationStartOptions) {\n workDurationMs ??= DEFAULT_WORK_DURATION_MS;\n sleepDurationMs ??= DEFAULT_SLEEP_DURATION_MS;\n\n this.running = true;\n while (this.running) {\n if (this.abortController.signal.aborted) {\n // We assign this here so that `stop()` can tell when this loop is done\n // processing jobs.\n this.running = false;\n return;\n }\n\n let didWork = false;\n try {\n didWork = await this.maybePerformWork(workDurationMs);\n } catch (err) {\n this.emit('error', err);\n }\n\n // If we did work, we'll immediately try again since there's probably more\n // work to be done. If not, we'll sleep for a while - maybe some more work\n // will become available!\n if (!didWork) {\n // We provide the signal here so that we can more quickly stop things\n // when we're shutting down.\n try {\n await sleep(sleepDurationMs, null, { ref: false, signal: this.abortController.signal });\n } catch {\n // We don't care about errors here, they should only ever occur when\n // the AbortController is aborted. Continue to the next iteration of\n // the loop so we can shut down.\n continue;\n }\n }\n }\n }\n\n private async getOrStartMigration(): Promise<BatchedMigrationRow | null> {\n return doWithLock(\n this.lockName,\n {\n // Don't fail if the lock couldn't be acquired immediately.\n onNotAcquired: () => null,\n },\n async () => {\n let migration = await queryOptionalRow(\n sql.select_running_migration,\n { project: this.options.project },\n BatchedMigrationRowSchema,\n );\n\n if (!migration) {\n migration = await queryOptionalRow(\n sql.start_next_pending_migration,\n { project: this.options.project },\n BatchedMigrationRowSchema,\n );\n }\n\n return migration;\n },\n );\n }\n\n async maybePerformWork(durationMs: number): Promise<boolean> {\n const migration = await this.getOrStartMigration();\n if (!migration) {\n // No work to do. Handle this case.\n return false;\n }\n\n // This server may not yet know about the current running migration. If\n // that's the case, we'll just skip it for now.\n const migrationFile = await this.getMigrationForIdentifier(migration.timestamp);\n if (!migrationFile) {\n return false;\n }\n\n let didWork = false;\n await doWithLock(\n this.lockNameForTimestamp(migrationFile.timestamp),\n {\n autoRenew: true,\n // Do nothing if the lock could not immediately be acquired.\n onNotAcquired: () => null,\n },\n async () => {\n didWork = true;\n const migrationImplementation = await this.loadMigrationImplementation(migrationFile);\n\n const runner = new BatchedMigrationRunner(migration, migrationImplementation);\n\n try {\n await runner.run({ signal: this.abortController.signal, durationMs });\n } catch (err) {\n this.emit('error', err);\n }\n },\n );\n\n return didWork;\n }\n\n async stop() {\n this.abortController.abort();\n\n // Spin until we're no longer running.\n while (this.running) {\n await sleep(1000);\n }\n }\n}\n\nlet runner: BatchedMigrationsRunner | null = null;\n\nfunction assertRunner(\n runner: BatchedMigrationsRunner | null,\n): asserts runner is BatchedMigrationsRunner {\n if (!runner) throw new Error('Batched migrations not initialized');\n}\n\nexport function initBatchedMigrations(options: BatchedMigrationRunnerOptions) {\n if (runner) throw new Error('Batched migrations already initialized');\n runner = new BatchedMigrationsRunner(options);\n return runner;\n}\n\nexport function startBatchedMigrations(options: BatchedMigrationStartOptions = {}) {\n assertRunner(runner);\n runner.start(options);\n return runner;\n}\n\nexport async function stopBatchedMigrations() {\n assertRunner(runner);\n await runner.stop();\n runner = null;\n}\n\n/**\n * Given a batched migration identifier like `20230406184103_migration`,\n * enqueues it for execution by creating a row in the `batched_migrations`\n * table.\n *\n * Despite taking a full identifier, only the timestamp is used to uniquely\n * identify the batched migration. The remaining part is just used to make\n * calls more human-readable.\n *\n * @param identifier The identifier of the batched migration to enqueue.\n */\nexport async function enqueueBatchedMigration(identifier: string) {\n assertRunner(runner);\n await runner.enqueueBatchedMigration(identifier);\n}\n\n/**\n * Given a batched migration identifier like `20230406184103_migration`,\n * synchronously runs it to completion. An error will be thrown if the final\n * status of the migration is not `succeeded`.\n *\n * @param identifier The identifier of the batched migration to finalize.\n * @param options Options for finalizing the batched migration.\n */\nexport async function finalizeBatchedMigration(\n identifier: string,\n options?: BatchedMigrationFinalizeOptions,\n) {\n assertRunner(runner);\n await runner.finalizeBatchedMigration(identifier, options);\n}\n"]}
package/dist/migrations/migrations.d.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { MigrationFile } from '../load-migrations.js';
1
+ import { type MigrationFile } from '../load-migrations.js';
2
2
  export declare function init(directories: string | string[], project: string): Promise<void>;
3
3
  export declare function getMigrationsToExecute(migrationFiles: MigrationFile[], executedMigrations: {
4
4
  timestamp: string | null;
package/dist/migrations/migrations.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/migrations/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,MAAM,UAAU,CAAC;AAE1B,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,wBAAwB,CAAC;AAEhD,OAAO,EAEL,gBAAgB,EAChB,wCAAwC,EACxC,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAErD,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,WAA8B,EAAE,OAAe;IACxE,MAAM,oBAAoB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC;IAC9B,MAAM,CAAC,OAAO,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;IAC/C,MAAM,UAAU,CAAC,UAAU,CACzB,QAAQ,EACR;QACE,wEAAwE;QACxE,mEAAmE;QACnE,wBAAwB;QACxB,EAAE;QACF,qEAAqE;QACrE,kEAAkE;QAClE,0EAA0E;QAC1E,SAAS,EAAE,IAAI;KAChB,EACD,KAAK,IAAI,EAAE;QACT,MAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC,CACF,CAAC;IACF,MAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,cAA+B,EAC/B,kBAAkD;IAElD,qDAAqD;IACrD,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACxF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,WAAqB,EAAE,OAAe;IACvE,MAAM,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAE/C,wCAAwC;IACxC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAExD,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,UAAU,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,UAAU,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,IAAI,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAE5E,MAAM,cAAc,GAAG,MAAM,wCAAwC,CAAC,WAAW,EAAE;QACjF,MAAM;QACN,KAAK;QACL,KAAK;QACL,MAAM;KACP,CAAC,CAAC;IAEH,4EAA4E;IAC5E,0EAA0E;IAC1E,kBAAkB;IAClB,MAAM,2BAA2B,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnF,IAAI,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb;YACE,kDAAkD;YAClD,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YACzD,kEAAkE;YAClE,qCAAqC;YACrC,0EAA0E;SAC3E,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IACJ,CAAC;IAED,oDAAoD;IACpD,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAExE,iDAAiD;IACjD,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAEhE,kDAAkD;IAClD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,oBAAoB,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAE7F,KAAK,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,mBAAmB,EAAE,CAAC;QACrE,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,iEAAiE;YACjE,MAAM,CAAC,OAAO,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACrD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;YACnD,IAAI,CAAC;gBACH,IAAI,WAAW,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC,qBAAqB,CAAC,KAAK,IAAI,EAAE;wBAC3C,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;oBAC3C,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC1C,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/C,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,qCAAqC,CAAC,CAAC;YAC9E,CAAC;YACD,MAAM,cAAc,EAAE,CAAC;QACzB,CAAC;QAED,wBAAwB;QACxB,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAC3C,QAAQ;YACR,SAAS;YACT,OAAO;SACR,CAAC,CAAC;IACL,CAAC;AACH,CAAC","sourcesContent":["import path from 'path';\n\nimport fs from 'fs-extra';\n\nimport * as error from '@prairielearn/error';\nimport { logger } from '@prairielearn/logger';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport * as sqldb from '@prairielearn/postgres';\n\nimport {\n MigrationFile,\n parseAnnotations,\n readAndValidateMigrationsFromDirectories,\n sortMigrationFiles,\n} from '../load-migrations.js';\n\nconst sql = sqldb.loadSqlEquiv(import.meta.filename);\n\nexport async function init(directories: string | string[], project: string) {\n const migrationDirectories = Array.isArray(directories) ? directories : [directories];\n const lockName = 'migrations';\n logger.verbose(`Waiting for lock ${lockName}`);\n await namedLocks.doWithLock(\n lockName,\n {\n // Migrations *might* take a long time to run, so we'll enable automatic\n // lock renewal so that our lock doesn't get killed by the Postgres\n // idle session timeout.\n //\n // That said, we should generally try to keep migrations executing as\n // quickly as possible. A long-running migration likely means that\n // Postgres is locking a whole table, which is unacceptable in production.\n autoRenew: true,\n },\n async () => {\n logger.verbose(`Acquired lock ${lockName}`);\n await initWithLock(migrationDirectories, project);\n },\n );\n logger.verbose(`Released lock ${lockName}`);\n}\n\nexport function getMigrationsToExecute(\n migrationFiles: MigrationFile[],\n executedMigrations: { timestamp: string | null }[],\n): MigrationFile[] {\n // If no migrations have ever been run, run them all.\n if (executedMigrations.length === 0) {\n return migrationFiles;\n }\n\n const executedMigrationTimestamps = new Set(executedMigrations.map((m) => m.timestamp));\n return migrationFiles.filter((m) => !executedMigrationTimestamps.has(m.timestamp));\n}\n\nexport async function initWithLock(directories: string[], project: string) {\n logger.verbose('Starting DB schema migration');\n\n // Create the migrations table if needed\n await sqldb.queryAsync(sql.create_migrations_table, {});\n\n // Apply necessary changes to the migrations table as needed.\n try {\n await sqldb.queryAsync('SELECT project FROM migrations;', {});\n } catch (err: any) {\n if (err.routine === 'errorMissingColumn') {\n logger.info('Altering migrations table');\n await sqldb.queryAsync(sql.add_projects_column, {});\n } else {\n throw err;\n }\n }\n try {\n await sqldb.queryAsync('SELECT timestamp FROM migrations;', {});\n } catch (err: any) {\n if (err.routine === 'errorMissingColumn') {\n logger.info('Altering migrations table again');\n await sqldb.queryAsync(sql.add_timestamp_column, {});\n } else {\n throw err;\n }\n }\n\n let allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });\n\n const migrationFiles = await readAndValidateMigrationsFromDirectories(directories, [\n '.sql',\n '.js',\n '.ts',\n '.mjs',\n ]);\n\n // Validation: if we not all previously-executed migrations have timestamps,\n // prompt the user to deploy an earlier version that includes both indexes\n // and timestamps.\n const migrationsMissingTimestamps = allMigrations.rows.filter((m) => !m.timestamp);\n if (migrationsMissingTimestamps.length > 0) {\n throw new Error(\n [\n 'The following migrations are missing timestamps:',\n migrationsMissingTimestamps.map((m) => ` ${m.filename}`),\n // This revision was the most recent commit to `master` before the\n // code handling indexes was removed.\n 'You must deploy revision 1aa43c7348fa24cf636413d720d06a2fa9e38ef2 first.',\n ].join('\\n'),\n );\n }\n\n // Refetch the list of migrations from the database.\n allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });\n\n // Sort the migration files into execution order.\n const sortedMigrationFiles = sortMigrationFiles(migrationFiles);\n\n // Figure out which migrations have to be applied.\n const migrationsToExecute = getMigrationsToExecute(sortedMigrationFiles, allMigrations.rows);\n\n for (const { directory, filename, timestamp } of migrationsToExecute) {\n if (allMigrations.rows.length === 0) {\n // if we are running all the migrations then log at a lower level\n logger.verbose(`Running migration ${filename}`);\n } else {\n logger.info(`Running migration ${filename}`);\n }\n\n const migrationPath = path.join(directory, filename);\n if (filename.endsWith('.sql')) {\n const migrationSql = await fs.readFile(migrationPath, 'utf8');\n const annotations = parseAnnotations(migrationSql);\n try {\n if (annotations.has('NO TRANSACTION')) {\n await sqldb.queryAsync(migrationSql, {});\n } else {\n await sqldb.runInTransactionAsync(async () => {\n await sqldb.queryAsync(migrationSql, {});\n });\n }\n } catch (err) {\n error.addData(err, { sqlFile: filename });\n throw err;\n }\n } else {\n const migrationModule = await import(migrationPath);\n const implementation = migrationModule.default;\n if (typeof implementation !== 'function') {\n throw new Error(`Migration ${filename} does not export a default function`);\n }\n await implementation();\n }\n\n // Record the migration.\n await sqldb.queryAsync(sql.insert_migration, {\n filename,\n timestamp,\n project,\n });\n }\n}\n"]}
1
+ {"version":3,"file":"migrations.js","sourceRoot":"","sources":["../../src/migrations/migrations.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AAExB,OAAO,EAAE,MAAM,UAAU,CAAC;AAE1B,OAAO,KAAK,KAAK,MAAM,qBAAqB,CAAC;AAC7C,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,KAAK,UAAU,MAAM,2BAA2B,CAAC;AACxD,OAAO,KAAK,KAAK,MAAM,wBAAwB,CAAC;AAEhD,OAAO,EAEL,gBAAgB,EAChB,wCAAwC,EACxC,kBAAkB,GACnB,MAAM,uBAAuB,CAAC;AAE/B,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAErD,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,WAA8B,EAAE,OAAe;IACxE,MAAM,oBAAoB,GAAG,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,YAAY,CAAC;IAC9B,MAAM,CAAC,OAAO,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;IAC/C,MAAM,UAAU,CAAC,UAAU,CACzB,QAAQ,EACR;QACE,wEAAwE;QACxE,mEAAmE;QACnE,wBAAwB;QACxB,EAAE;QACF,qEAAqE;QACrE,kEAAkE;QAClE,0EAA0E;QAC1E,SAAS,EAAE,IAAI;KAChB,EACD,KAAK,IAAI,EAAE;QACT,MAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,YAAY,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAC;IACpD,CAAC,CACF,CAAC;IACF,MAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,cAA+B,EAC/B,kBAAkD;IAElD,qDAAqD;IACrD,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,cAAc,CAAC;IACxB,CAAC;IAED,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACxF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,WAAqB,EAAE,OAAe;IACvE,MAAM,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAE/C,wCAAwC;IACxC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAExD,6DAA6D;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,UAAU,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;IAChE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;QACtD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IACD,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,UAAU,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE,CAAC;YACzC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;QACvD,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;IAED,IAAI,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAE5E,MAAM,cAAc,GAAG,MAAM,wCAAwC,CAAC,WAAW,EAAE;QACjF,MAAM;QACN,KAAK;QACL,KAAK;QACL,MAAM;KACP,CAAC,CAAC;IAEH,4EAA4E;IAC5E,0EAA0E;IAC1E,kBAAkB;IAClB,MAAM,2BAA2B,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnF,IAAI,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb;YACE,kDAAkD;YAClD,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YACzD,kEAAkE;YAClE,qCAAqC;YACrC,0EAA0E;SAC3E,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;IACJ,CAAC;IAED,oDAAoD;IACpD,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAExE,iDAAiD;IACjD,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAEhE,kDAAkD;IAClD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,oBAAoB,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAE7F,KAAK,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,mBAAmB,EAAE,CAAC;QACrE,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpC,iEAAiE;YACjE,MAAM,CAAC,OAAO,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;QAC/C,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QACrD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAC9D,MAAM,WAAW,GAAG,gBAAgB,CAAC,YAAY,CAAC,CAAC;YACnD,IAAI,CAAC;gBACH,IAAI,WAAW,CAAC,GAAG,CAAC,gBAAgB,CAAC,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;gBAC3C,CAAC;qBAAM,CAAC;oBACN,MAAM,KAAK,CAAC,qBAAqB,CAAC,KAAK,IAAI,EAAE;wBAC3C,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;oBAC3C,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC1C,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,eAAe,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;YACpD,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/C,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,qCAAqC,CAAC,CAAC;YAC9E,CAAC;YACD,MAAM,cAAc,EAAE,CAAC;QACzB,CAAC;QAED,wBAAwB;QACxB,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAC3C,QAAQ;YACR,SAAS;YACT,OAAO;SACR,CAAC,CAAC;IACL,CAAC;AACH,CAAC","sourcesContent":["import path from 'path';\n\nimport fs from 'fs-extra';\n\nimport * as error from '@prairielearn/error';\nimport { logger } from '@prairielearn/logger';\nimport * as namedLocks from '@prairielearn/named-locks';\nimport * as sqldb from '@prairielearn/postgres';\n\nimport {\n type MigrationFile,\n parseAnnotations,\n readAndValidateMigrationsFromDirectories,\n sortMigrationFiles,\n} from '../load-migrations.js';\n\nconst sql = sqldb.loadSqlEquiv(import.meta.filename);\n\nexport async function init(directories: string | string[], project: string) {\n const migrationDirectories = Array.isArray(directories) ? directories : [directories];\n const lockName = 'migrations';\n logger.verbose(`Waiting for lock ${lockName}`);\n await namedLocks.doWithLock(\n lockName,\n {\n // Migrations *might* take a long time to run, so we'll enable automatic\n // lock renewal so that our lock doesn't get killed by the Postgres\n // idle session timeout.\n //\n // That said, we should generally try to keep migrations executing as\n // quickly as possible. A long-running migration likely means that\n // Postgres is locking a whole table, which is unacceptable in production.\n autoRenew: true,\n },\n async () => {\n logger.verbose(`Acquired lock ${lockName}`);\n await initWithLock(migrationDirectories, project);\n },\n );\n logger.verbose(`Released lock ${lockName}`);\n}\n\nexport function getMigrationsToExecute(\n migrationFiles: MigrationFile[],\n executedMigrations: { timestamp: string | null }[],\n): MigrationFile[] {\n // If no migrations have ever been run, run them all.\n if (executedMigrations.length === 0) {\n return migrationFiles;\n }\n\n const executedMigrationTimestamps = new Set(executedMigrations.map((m) => m.timestamp));\n return migrationFiles.filter((m) => !executedMigrationTimestamps.has(m.timestamp));\n}\n\nexport async function initWithLock(directories: string[], project: string) {\n logger.verbose('Starting DB schema migration');\n\n // Create the migrations table if needed\n await sqldb.queryAsync(sql.create_migrations_table, {});\n\n // Apply necessary changes to the migrations table as needed.\n try {\n await sqldb.queryAsync('SELECT project FROM migrations;', {});\n } catch (err: any) {\n if (err.routine === 'errorMissingColumn') {\n logger.info('Altering migrations table');\n await sqldb.queryAsync(sql.add_projects_column, {});\n } else {\n throw err;\n }\n }\n try {\n await sqldb.queryAsync('SELECT timestamp FROM migrations;', {});\n } catch (err: any) {\n if (err.routine === 'errorMissingColumn') {\n logger.info('Altering migrations table again');\n await sqldb.queryAsync(sql.add_timestamp_column, {});\n } else {\n throw err;\n }\n }\n\n let allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });\n\n const migrationFiles = await readAndValidateMigrationsFromDirectories(directories, [\n '.sql',\n '.js',\n '.ts',\n '.mjs',\n ]);\n\n // Validation: if we not all previously-executed migrations have timestamps,\n // prompt the user to deploy an earlier version that includes both indexes\n // and timestamps.\n const migrationsMissingTimestamps = allMigrations.rows.filter((m) => !m.timestamp);\n if (migrationsMissingTimestamps.length > 0) {\n throw new Error(\n [\n 'The following migrations are missing timestamps:',\n migrationsMissingTimestamps.map((m) => ` ${m.filename}`),\n // This revision was the most recent commit to `master` before the\n // code handling indexes was removed.\n 'You must deploy revision 1aa43c7348fa24cf636413d720d06a2fa9e38ef2 first.',\n ].join('\\n'),\n );\n }\n\n // Refetch the list of migrations from the database.\n allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });\n\n // Sort the migration files into execution order.\n const sortedMigrationFiles = sortMigrationFiles(migrationFiles);\n\n // Figure out which migrations have to be applied.\n const migrationsToExecute = getMigrationsToExecute(sortedMigrationFiles, allMigrations.rows);\n\n for (const { directory, filename, timestamp } of migrationsToExecute) {\n if (allMigrations.rows.length === 0) {\n // if we are running all the migrations then log at a lower level\n logger.verbose(`Running migration ${filename}`);\n } else {\n logger.info(`Running migration ${filename}`);\n }\n\n const migrationPath = path.join(directory, filename);\n if (filename.endsWith('.sql')) {\n const migrationSql = await fs.readFile(migrationPath, 'utf8');\n const annotations = parseAnnotations(migrationSql);\n try {\n if (annotations.has('NO TRANSACTION')) {\n await sqldb.queryAsync(migrationSql, {});\n } else {\n await sqldb.runInTransactionAsync(async () => {\n await sqldb.queryAsync(migrationSql, {});\n });\n }\n } catch (err) {\n error.addData(err, { sqlFile: filename });\n throw err;\n }\n } else {\n const migrationModule = await import(migrationPath);\n const implementation = migrationModule.default;\n if (typeof implementation !== 'function') {\n throw new Error(`Migration ${filename} does not export a default function`);\n }\n await implementation();\n }\n\n // Record the migration.\n await sqldb.queryAsync(sql.insert_migration, {\n filename,\n timestamp,\n project,\n });\n }\n}\n"]}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@prairielearn/migrations",
3
- "version": "3.0.7",
3
+ "version": "3.0.9",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "repository": {
@@ -14,10 +14,10 @@
14
14
  "test": "c8 mocha src/**/*.test.ts"
15
15
  },
16
16
  "dependencies": {
17
- "@prairielearn/error": "^2.0.5",
18
- "@prairielearn/logger": "^2.0.5",
19
- "@prairielearn/named-locks": "^3.0.7",
20
- "@prairielearn/postgres": "^2.1.3",
17
+ "@prairielearn/error": "^2.0.7",
18
+ "@prairielearn/logger": "^2.0.7",
19
+ "@prairielearn/named-locks": "^3.0.9",
20
+ "@prairielearn/postgres": "^2.1.5",
21
21
  "fs-extra": "^11.2.0",
22
22
  "serialize-error": "^11.0.3",
23
23
  "zod": "^3.23.8"
@@ -25,15 +25,15 @@
25
25
  "devDependencies": {
26
26
  "@prairielearn/tsconfig": "^0.0.0",
27
27
  "@types/fs-extra": "^11.0.4",
28
- "@types/mocha": "^10.0.7",
29
- "@types/node": "^20.14.15",
28
+ "@types/mocha": "^10.0.10",
29
+ "@types/node": "^20.17.9",
30
30
  "c8": "^10.1.2",
31
- "chai": "^5.1.1",
32
- "chai-as-promised": "^8.0.0",
33
- "mocha": "^10.7.3",
31
+ "chai": "^5.1.2",
32
+ "chai-as-promised": "^8.0.1",
33
+ "mocha": "^10.8.2",
34
34
  "tmp-promise": "^3.0.3",
35
- "tsx": "^4.17.0",
36
- "typescript": "^5.5.4",
35
+ "tsx": "^4.19.2",
36
+ "typescript": "^5.7.2",
37
37
  "typescript-cp": "^0.1.9"
38
38
  },
39
39
  "c8": {
package/src/batched-migrations/batched-migration-runner.ts CHANGED
@@ -6,15 +6,15 @@ import { loadSqlEquiv, queryAsync, queryRow, queryOptionalRow } from '@prairiele
6
6
 
7
7
  import {
8
8
  BatchedMigrationJobRowSchema,
9
- BatchedMigrationJobStatus,
10
- BatchedMigrationJobRow,
9
+ type BatchedMigrationJobStatus,
10
+ type BatchedMigrationJobRow,
11
11
  } from './batched-migration-job.js';
12
12
  import {
13
- BatchedMigrationStatus,
14
- BatchedMigrationRow,
13
+ type BatchedMigrationStatus,
14
+ type BatchedMigrationRow,
15
15
  updateBatchedMigrationStatus,
16
16
  BatchedMigrationStatusSchema,
17
- BatchedMigrationImplementation,
17
+ type BatchedMigrationImplementation,
18
18
  } from './batched-migration.js';
19
19
 
20
20
  const sql = loadSqlEquiv(import.meta.filename);
package/src/batched-migrations/batched-migrations-runner.ts CHANGED
@@ -5,17 +5,20 @@ import { setTimeout as sleep } from 'node:timers/promises';
5
5
  import { doWithLock } from '@prairielearn/named-locks';
6
6
  import { loadSqlEquiv, queryOptionalRow } from '@prairielearn/postgres';
7
7
 
8
- import { MigrationFile, readAndValidateMigrationsFromDirectories } from '../load-migrations.js';
8
+ import {
9
+ type MigrationFile,
10
+ readAndValidateMigrationsFromDirectories,
11
+ } from '../load-migrations.js';
9
12
 
10
13
  import { BatchedMigrationRunner } from './batched-migration-runner.js';
11
14
  import {
12
15
  BatchedMigrationRowSchema,
13
- BatchedMigrationRow,
16
+ type BatchedMigrationRow,
14
17
  insertBatchedMigration,
15
- BatchedMigrationStatus,
18
+ type BatchedMigrationStatus,
16
19
  selectBatchedMigrationForTimestamp,
17
20
  updateBatchedMigrationStatus,
18
- BatchedMigrationImplementation,
21
+ type BatchedMigrationImplementation,
19
22
  validateBatchedMigrationImplementation,
20
23
  } from './batched-migration.js';
21
24
 
@@ -195,7 +198,7 @@ export class BatchedMigrationsRunner extends EventEmitter {
195
198
  // when we're shutting down.
196
199
  try {
197
200
  await sleep(sleepDurationMs, null, { ref: false, signal: this.abortController.signal });
198
- } catch (err) {
201
+ } catch {
199
202
  // We don't care about errors here, they should only ever occur when
200
203
  // the AbortController is aborted. Continue to the next iteration of
201
204
  // the loop so we can shut down.
package/src/migrations/migrations.ts CHANGED
@@ -8,7 +8,7 @@ import * as namedLocks from '@prairielearn/named-locks';
8
8
  import * as sqldb from '@prairielearn/postgres';
9
9
 
10
10
  import {
11
- MigrationFile,
11
+ type MigrationFile,
12
12
  parseAnnotations,
13
13
  readAndValidateMigrationsFromDirectories,
14
14
  sortMigrationFiles,