@prairielearn/migrations 3.0.4 → 3.0.5

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # @prairielearn/migrations
 
+## 3.0.5
+
+### Patch Changes
+
+- 24a93b8: Upgrade all JavaScript dependencies
+- Updated dependencies [24a93b8]
+  - @prairielearn/named-locks@3.0.5
+  - @prairielearn/postgres@2.1.1
+  - @prairielearn/logger@2.0.3
+  - @prairielearn/error@2.0.3
+
 ## 3.0.4
 
 ### Patch Changes

package/dist/batched-migrations/batched-migration-runner.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" resolution-mode="require"/>
 import { BatchedMigrationRow, BatchedMigrationImplementation } from './batched-migration.js';
 interface BatchedMigrationRunnerOptions {
     logProgress?: boolean;

package/dist/batched-migrations/batched-migration-runner.js

@@ -36,7 +36,7 @@ export class BatchedMigrationRunner {
         // Safety check: if there are any pending jobs, don't mark this
         // migration as finished.
         if (await this.hasIncompleteJobs(migration)) {
-            this.log(`Incomplete jobs found, not marking as finished`);
+            this.log('Incomplete jobs found, not marking as finished');
             return;
         }
         const hasFailedJobs = await this.hasFailedJobs(migration);

package/dist/batched-migrations/batched-migration-runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,4BAA4B,GAG7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAGL,4BAA4B,EAC5B,4BAA4B,GAE7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB;YACE,EAAE,EAAE,SAAS,CAAC,EAAE;SACjB,EACD,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryRow, queryOptionalRow } from '@prairielearn/postgres';\n\nimport {\n  BatchedMigrationJobRowSchema,\n  BatchedMigrationJobStatus,\n  BatchedMigrationJobRow,\n} from './batched-migration-job.js';\nimport {\n  BatchedMigrationStatus,\n  BatchedMigrationRow,\n  updateBatchedMigrationStatus,\n  BatchedMigrationStatusSchema,\n  BatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n  logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n  private options: BatchedMigrationRunnerOptions;\n  private migration: BatchedMigrationRow;\n  private migrationImplementation: BatchedMigrationImplementation;\n  private migrationStatus: BatchedMigrationStatus;\n\n  constructor(\n    migration: BatchedMigrationRow,\n    migrationImplementation: BatchedMigrationImplementation,\n    options: BatchedMigrationRunnerOptions = {},\n  ) {\n    this.options = options;\n    this.migration = migration;\n    this.migrationImplementation = migrationImplementation;\n    this.migrationStatus = migration.status;\n  }\n\n  private log(message: string, ...meta: any[]) {\n    if (this.options.logProgress) {\n      logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n    }\n  }\n\n  private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n    return await queryRow(\n      sql.batched_migration_has_incomplete_jobs,\n      { batched_migration_id: migration.id },\n      z.boolean(),\n    );\n  }\n\n  private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n    return await queryRow(\n      sql.batched_migration_has_failed_jobs,\n      { batched_migration_id: migration.id },\n      z.boolean(),\n    );\n  }\n\n  private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n    this.migrationStatus = await queryRow(\n      sql.get_migration_status,\n      {\n        id: migration.id,\n      },\n      BatchedMigrationStatusSchema,\n    );\n  }\n\n  private async finishRunningMigration(migration: BatchedMigrationRow) {\n    // Safety check: if there are any pending jobs, don't mark this\n    // migration as finished.\n    if (await this.hasIncompleteJobs(migration)) {\n      this.log(`Incomplete jobs found, not marking as finished`);\n      return;\n    }\n\n    const hasFailedJobs = await this.hasFailedJobs(migration);\n    const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n    await updateBatchedMigrationStatus(migration.id, finalStatus);\n    this.log(`Finished with status '${finalStatus}'`);\n  }\n\n  private async getNextBatchBounds(\n    migration: BatchedMigrationRow,\n  ): Promise<null | [bigint, bigint]> {\n    const lastJob = await queryOptionalRow(\n      sql.select_last_batched_migration_job,\n      { batched_migration_id: migration.id },\n      BatchedMigrationJobRowSchema,\n    );\n\n    const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n    if (nextMin > migration.max_value) return null;\n\n    let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n    if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n    return [nextMin, nextMax];\n  }\n\n  private async startJob(job: BatchedMigrationJobRow) {\n    await queryAsync(sql.start_batched_migration_job, { id: job.id });\n    const jobRange = `[${job.min_value}, ${job.max_value}]`;\n    const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n    this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n  }\n\n  private serializeJobData(data: unknown) {\n    if (data == null) return null;\n\n    // Return JSON-stringified data. Convert BigInts to strings.\n    return JSON.stringify(data, (_key, value) => {\n      if (typeof value === 'bigint') return value.toString();\n      return value;\n    });\n  }\n\n  private async finishJob(\n    job: BatchedMigrationJobRow,\n    status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n    data?: unknown,\n  ) {\n    await queryAsync(sql.finish_batched_migration_job, {\n      id: job.id,\n      status,\n      data: this.serializeJobData(data),\n    });\n    this.log(`Job ${job.id} finished with status '${status}'`);\n  }\n\n  private async getOrCreateNextMigrationJob(\n    migration: BatchedMigrationRow,\n  ): Promise<BatchedMigrationJobRow | null> {\n    const nextBatchBounds = await this.getNextBatchBounds(migration);\n    if (nextBatchBounds) {\n      return await queryRow(\n        sql.insert_batched_migration_job,\n        {\n          batched_migration_id: migration.id,\n          min_value: nextBatchBounds[0],\n          max_value: nextBatchBounds[1],\n        },\n        BatchedMigrationJobRowSchema,\n      );\n    } else {\n      // Pick up any old pending jobs from this migration. These will only exist if\n      // an admin manually elected to retry all failed jobs; we'll never automatically\n      // transition failed jobs back to pending.\n      return await queryOptionalRow(\n        sql.select_first_pending_batched_migration_job,\n        { batched_migration_id: migration.id },\n        BatchedMigrationJobRowSchema,\n      );\n    }\n  }\n\n  private async runMigrationJob(\n    migration: BatchedMigrationRow,\n    migrationImplementation: BatchedMigrationImplementation,\n  ) {\n    const nextJob = await this.getOrCreateNextMigrationJob(migration);\n    if (nextJob) {\n      await this.startJob(nextJob);\n\n      let error = null;\n      try {\n        // We'll only handle errors thrown by the migration itself. If any of\n        // our own execution machinery throws an error, we'll let it bubble up.\n        await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n      } catch (err) {\n        error = err;\n      }\n\n      if (error) {\n        logger.error(\n          `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n          error,\n        );\n        await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n      } else {\n        await this.finishJob(nextJob, 'succeeded');\n      }\n    } else {\n      await this.finishRunningMigration(migration);\n    }\n  }\n\n  async run({\n    signal,\n    iterations,\n    durationMs,\n  }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n    let iterationCount = 0;\n    const endTime = durationMs ? Date.now() + durationMs : null;\n    while (\n      !signal?.aborted &&\n      (iterations ? iterationCount < iterations : true) &&\n      (endTime ? Date.now() < endTime : true) &&\n      (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n    ) {\n      await this.runMigrationJob(this.migration, this.migrationImplementation);\n      iterationCount += 1;\n      // Always refresh the status so we can detect if the migration was marked\n      // as paused by another process.\n      await this.refreshMigrationStatus(this.migration);\n    }\n  }\n}\n"]}
+{"version":3,"file":"batched-migration-runner.js","sourceRoot":"","sources":["../../src/batched-migrations/batched-migration-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,4BAA4B,GAG7B,MAAM,4BAA4B,CAAC;AACpC,OAAO,EAGL,4BAA4B,EAC5B,4BAA4B,GAE7B,MAAM,wBAAwB,CAAC;AAEhC,MAAM,GAAG,GAAG,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;AAM/C,MAAM,OAAO,sBAAsB;IACzB,OAAO,CAAgC;IACvC,SAAS,CAAsB;IAC/B,uBAAuB,CAAiC;IACxD,eAAe,CAAyB;IAEhD,YACE,SAA8B,EAC9B,uBAAuD,EACvD,UAAyC,EAAE;QAE3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,uBAAuB,GAAG,uBAAuB,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC;IAC1C,CAAC;IAEO,GAAG,CAAC,OAAe,EAAE,GAAG,IAAW;QACzC,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,EAAE,EAAE,GAAG,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,SAA8B;QAC5D,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,qCAAqC,EACzC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAA8B;QACxD,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,IAAI,CAAC,eAAe,GAAG,MAAM,QAAQ,CACnC,GAAG,CAAC,oBAAoB,EACxB;YACE,EAAE,EAAE,SAAS,CAAC,EAAE;SACjB,EACD,4BAA4B,CAC7B,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,sBAAsB,CAAC,SAA8B;QACjE,+DAA+D;QAC/D,yBAAyB;QACzB,IAAI,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,OAAO;QACT,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,WAAW,GAAG,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC;QAC3D,MAAM,4BAA4B,CAAC,SAAS,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;QAC9D,IAAI,CAAC,GAAG,CAAC,yBAAyB,WAAW,GAAG,CAAC,CAAC;IACpD,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,SAA8B;QAE9B,MAAM,OAAO,GAAG,MAAM,gBAAgB,CACpC,GAAG,CAAC,iCAAiC,EACrC,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC;QACvE,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE/C,IAAI,OAAO,GAAG,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QAC1D,IAAI,OAAO,GAAG,SAAS,CAAC,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC;QAEjE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,GAA2B;QAChD,MAAM,UAAU,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,KAAK,GAAG,CAAC,SAAS,GAAG,CAAC;QACxD,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,SAAS,GAAG,CAAC;QACpF,IAAI,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,EAAE,cAAc,QAAQ,OAAO,cAAc,EAAE,CAAC,CAAC;IAC/E,CAAC;IAEO,gBAAgB,CAAC,IAAa;QACpC,IAAI,IAAI,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QAE9B,4DAA4D;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;YAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;gBAAE,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;YACvD,OAAO,KAAK,CAAC;QACf,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,GAA2B,EAC3B,MAAkE,EAClE,IAAc;QAEd,MAAM,UAAU,CAAC,GAAG,CAAC,4BAA4B,EAAE;YACjD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM;YACN,IAAI,EAAE,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,EAAE,0BAA0B,MAAM,GAAG,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,2BAA2B,CACvC,SAA8B;QAE9B,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;QACjE,IAAI,eAAe,EAAE,CAAC;YACpB,OAAO,MAAM,QAAQ,CACnB,GAAG,CAAC,4BAA4B,EAChC;gBACE,oBAAoB,EAAE,SAAS,CAAC,EAAE;gBAClC,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;gBAC7B,SAAS,EAAE,eAAe,CAAC,CAAC,CAAC;aAC9B,EACD,4BAA4B,CAC7B,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,6EAA6E;YAC7E,gFAAgF;YAChF,0CAA0C;YAC1C,OAAO,MAAM,gBAAgB,CAC3B,GAAG,CAAC,0CAA0C,EAC9C,EAAE,oBAAoB,EAAE,SAAS,CAAC,EAAE,EAAE,EACtC,4BAA4B,CAC7B,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,SAA8B,EAC9B,uBAAuD;QAEvD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,2BAA2B,CAAC,SAAS,CAAC,CAAC;QAClE,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAE7B,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,IAAI,CAAC;gBACH,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,uBAAuB,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;YAC9E,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,KAAK,GAAG,GAAG,CAAC;YACd,CAAC;YAED,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CACV,qBAAqB,OAAO,CAAC,EAAE,0BAA0B,SAAS,CAAC,QAAQ,EAAE,EAC7E,KAAK,CACN,CAAC;gBACF,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC5E,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,sBAAsB,CAAC,SAAS,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EACR,MAAM,EACN,UAAU,EACV,UAAU,MAC4D,EAAE;QACxE,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;QAC5D,OACE,CAAC,MAAM,EAAE,OAAO;YAChB,CAAC,UAAU,CAAC,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC;YACjD,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;YACvC,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS,IAAI,IAAI,CAAC,eAAe,KAAK,YAAY,CAAC,EAC7E,CAAC;YACD,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,uBAAuB,CAAC,CAAC;YACzE,cAAc,IAAI,CAAC,CAAC;YACpB,yEAAyE;YACzE,gCAAgC;YAChC,MAAM,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;CACF","sourcesContent":["import { serializeError } from 'serialize-error';\nimport { z } from 'zod';\n\nimport { logger } from '@prairielearn/logger';\nimport { loadSqlEquiv, queryAsync, queryRow, queryOptionalRow } from '@prairielearn/postgres';\n\nimport {\n  BatchedMigrationJobRowSchema,\n  BatchedMigrationJobStatus,\n  BatchedMigrationJobRow,\n} from './batched-migration-job.js';\nimport {\n  BatchedMigrationStatus,\n  BatchedMigrationRow,\n  updateBatchedMigrationStatus,\n  BatchedMigrationStatusSchema,\n  BatchedMigrationImplementation,\n} from './batched-migration.js';\n\nconst sql = loadSqlEquiv(import.meta.filename);\n\ninterface BatchedMigrationRunnerOptions {\n  logProgress?: boolean;\n}\n\nexport class BatchedMigrationRunner {\n  private options: BatchedMigrationRunnerOptions;\n  private migration: BatchedMigrationRow;\n  private migrationImplementation: BatchedMigrationImplementation;\n  private migrationStatus: BatchedMigrationStatus;\n\n  constructor(\n    migration: BatchedMigrationRow,\n    migrationImplementation: BatchedMigrationImplementation,\n    options: BatchedMigrationRunnerOptions = {},\n  ) {\n    this.options = options;\n    this.migration = migration;\n    this.migrationImplementation = migrationImplementation;\n    this.migrationStatus = migration.status;\n  }\n\n  private log(message: string, ...meta: any[]) {\n    if (this.options.logProgress) {\n      logger.info(`[${this.migration.filename}] ${message}`, ...meta);\n    }\n  }\n\n  private async hasIncompleteJobs(migration: BatchedMigrationRow): Promise<boolean> {\n    return await queryRow(\n      sql.batched_migration_has_incomplete_jobs,\n      { batched_migration_id: migration.id },\n      z.boolean(),\n    );\n  }\n\n  private async hasFailedJobs(migration: BatchedMigrationRow): Promise<boolean> {\n    return await queryRow(\n      sql.batched_migration_has_failed_jobs,\n      { batched_migration_id: migration.id },\n      z.boolean(),\n    );\n  }\n\n  private async refreshMigrationStatus(migration: BatchedMigrationRow) {\n    this.migrationStatus = await queryRow(\n      sql.get_migration_status,\n      {\n        id: migration.id,\n      },\n      BatchedMigrationStatusSchema,\n    );\n  }\n\n  private async finishRunningMigration(migration: BatchedMigrationRow) {\n    // Safety check: if there are any pending jobs, don't mark this\n    // migration as finished.\n    if (await this.hasIncompleteJobs(migration)) {\n      this.log('Incomplete jobs found, not marking as finished');\n      return;\n    }\n\n    const hasFailedJobs = await this.hasFailedJobs(migration);\n    const finalStatus = hasFailedJobs ? 'failed' : 'succeeded';\n    await updateBatchedMigrationStatus(migration.id, finalStatus);\n    this.log(`Finished with status '${finalStatus}'`);\n  }\n\n  private async getNextBatchBounds(\n    migration: BatchedMigrationRow,\n  ): Promise<null | [bigint, bigint]> {\n    const lastJob = await queryOptionalRow(\n      sql.select_last_batched_migration_job,\n      { batched_migration_id: migration.id },\n      BatchedMigrationJobRowSchema,\n    );\n\n    const nextMin = lastJob ? lastJob.max_value + 1n : migration.min_value;\n    if (nextMin > migration.max_value) return null;\n\n    let nextMax = nextMin + BigInt(migration.batch_size) - 1n;\n    if (nextMax > migration.max_value) nextMax = migration.max_value;\n\n    return [nextMin, nextMax];\n  }\n\n  private async startJob(job: BatchedMigrationJobRow) {\n    await queryAsync(sql.start_batched_migration_job, { id: job.id });\n    const jobRange = `[${job.min_value}, ${job.max_value}]`;\n    const migrationRange = `[${this.migration.min_value}, ${this.migration.max_value}]`;\n    this.log(`Started job ${job.id} for range ${jobRange} in ${migrationRange}`);\n  }\n\n  private serializeJobData(data: unknown) {\n    if (data == null) return null;\n\n    // Return JSON-stringified data. Convert BigInts to strings.\n    return JSON.stringify(data, (_key, value) => {\n      if (typeof value === 'bigint') return value.toString();\n      return value;\n    });\n  }\n\n  private async finishJob(\n    job: BatchedMigrationJobRow,\n    status: Extract<BatchedMigrationJobStatus, 'failed' | 'succeeded'>,\n    data?: unknown,\n  ) {\n    await queryAsync(sql.finish_batched_migration_job, {\n      id: job.id,\n      status,\n      data: this.serializeJobData(data),\n    });\n    this.log(`Job ${job.id} finished with status '${status}'`);\n  }\n\n  private async getOrCreateNextMigrationJob(\n    migration: BatchedMigrationRow,\n  ): Promise<BatchedMigrationJobRow | null> {\n    const nextBatchBounds = await this.getNextBatchBounds(migration);\n    if (nextBatchBounds) {\n      return await queryRow(\n        sql.insert_batched_migration_job,\n        {\n          batched_migration_id: migration.id,\n          min_value: nextBatchBounds[0],\n          max_value: nextBatchBounds[1],\n        },\n        BatchedMigrationJobRowSchema,\n      );\n    } else {\n      // Pick up any old pending jobs from this migration. These will only exist if\n      // an admin manually elected to retry all failed jobs; we'll never automatically\n      // transition failed jobs back to pending.\n      return await queryOptionalRow(\n        sql.select_first_pending_batched_migration_job,\n        { batched_migration_id: migration.id },\n        BatchedMigrationJobRowSchema,\n      );\n    }\n  }\n\n  private async runMigrationJob(\n    migration: BatchedMigrationRow,\n    migrationImplementation: BatchedMigrationImplementation,\n  ) {\n    const nextJob = await this.getOrCreateNextMigrationJob(migration);\n    if (nextJob) {\n      await this.startJob(nextJob);\n\n      let error = null;\n      try {\n        // We'll only handle errors thrown by the migration itself. If any of\n        // our own execution machinery throws an error, we'll let it bubble up.\n        await migrationImplementation.execute(nextJob.min_value, nextJob.max_value);\n      } catch (err) {\n        error = err;\n      }\n\n      if (error) {\n        logger.error(\n          `Error running job ${nextJob.id} for batched migration ${migration.filename}`,\n          error,\n        );\n        await this.finishJob(nextJob, 'failed', { error: serializeError(error) });\n      } else {\n        await this.finishJob(nextJob, 'succeeded');\n      }\n    } else {\n      await this.finishRunningMigration(migration);\n    }\n  }\n\n  async run({\n    signal,\n    iterations,\n    durationMs,\n  }: { signal?: AbortSignal; iterations?: number; durationMs?: number } = {}) {\n    let iterationCount = 0;\n    const endTime = durationMs ? Date.now() + durationMs : null;\n    while (\n      !signal?.aborted &&\n      (iterations ? iterationCount < iterations : true) &&\n      (endTime ? Date.now() < endTime : true) &&\n      (this.migrationStatus === 'running' || this.migrationStatus === 'finalizing')\n    ) {\n      await this.runMigrationJob(this.migration, this.migrationImplementation);\n      iterationCount += 1;\n      // Always refresh the status so we can detect if the migration was marked\n      // as paused by another process.\n      await this.refreshMigrationStatus(this.migration);\n    }\n  }\n}\n"]}

package/dist/batched-migrations/batched-migrations-runner.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" resolution-mode="require"/>
 import { EventEmitter } from 'node:events';
 interface BatchedMigrationRunnerOptions {
     project: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@prairielearn/migrations",
-  "version": "3.0.4",
+  "version": "3.0.5",
   "type": "module",
   "main": "./dist/index.js",
   "repository": {
@@ -14,10 +14,10 @@
     "test": "c8 mocha src/**/*.test.ts"
   },
   "dependencies": {
-    "@prairielearn/error": "^2.0.2",
-    "@prairielearn/logger": "^2.0.2",
-    "@prairielearn/named-locks": "^3.0.4",
-    "@prairielearn/postgres": "^2.1.0",
+    "@prairielearn/error": "^2.0.3",
+    "@prairielearn/logger": "^2.0.3",
+    "@prairielearn/named-locks": "^3.0.5",
+    "@prairielearn/postgres": "^2.1.1",
     "fs-extra": "^11.2.0",
     "serialize-error": "^11.0.3",
     "zod": "^3.23.8"
@@ -25,15 +25,15 @@
   "devDependencies": {
     "@prairielearn/tsconfig": "^0.0.0",
     "@types/fs-extra": "^11.0.4",
-    "@types/mocha": "^10.0.6",
-    "@types/node": "^20.13.0",
-    "c8": "^9.1.0",
+    "@types/mocha": "^10.0.7",
+    "@types/node": "^20.14.9",
+    "c8": "^10.1.2",
     "chai": "^5.1.1",
-    "chai-as-promised": "^7.1.2",
-    "mocha": "^10.4.0",
+    "chai-as-promised": "^8.0.0",
+    "mocha": "^10.6.0",
     "tmp-promise": "^3.0.3",
-    "tsx": "^4.11.0",
-    "typescript": "^5.4.5",
+    "tsx": "^4.16.2",
+    "typescript": "^5.5.3",
     "typescript-cp": "^0.1.9"
   },
   "c8": {

package/src/batched-migrations/batched-migration-runner.ts

@@ -76,7 +76,7 @@ export class BatchedMigrationRunner {
     // Safety check: if there are any pending jobs, don't mark this
     // migration as finished.
     if (await this.hasIncompleteJobs(migration)) {
-      this.log(`Incomplete jobs found, not marking as finished`);
+      this.log('Incomplete jobs found, not marking as finished');
       return;
     }