@prairielearn/migrations 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/dist/fixtures/20230407210409_create_users.sql +2 -0
- package/dist/fixtures/20230407210430_insert_user.d.ts +1 -0
- package/dist/fixtures/20230407210430_insert_user.js +7 -0
- package/dist/fixtures/20230407210430_insert_user.js.map +1 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +40 -15
- package/dist/index.js.map +1 -1
- package/dist/index.test.js +23 -2
- package/dist/index.test.js.map +1 -1
- package/package.json +9 -4
- package/src/fixtures/20230407210409_create_users.sql +2 -0
- package/src/fixtures/20230407210430_insert_user.ts +5 -0
- package/src/index.test.ts +29 -2
- package/src/index.ts +47 -19
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# @prairielearn/migrations
|
|
2
|
+
|
|
3
|
+
## 1.1.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 400a0b901: Use automatically-renewing named lock
|
|
8
|
+
- 751010ea3: Support JavaScript migration files
|
|
9
|
+
|
|
10
|
+
### Patch Changes
|
|
11
|
+
|
|
12
|
+
- Updated dependencies [400a0b901]
|
|
13
|
+
- @prairielearn/named-locks@1.2.0
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
const postgres_1 = require("@prairielearn/postgres");
|
|
4
|
+
module.exports = async function migrate() {
|
|
5
|
+
await (0, postgres_1.queryAsync)("INSERT INTO users (name) VALUES ('Test User')", {});
|
|
6
|
+
};
|
|
7
|
+
//# sourceMappingURL=20230407210430_insert_user.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"20230407210430_insert_user.js","sourceRoot":"","sources":["../../src/fixtures/20230407210430_insert_user.ts"],"names":[],"mappings":";;AAAA,qDAAoD;AAEpD,MAAM,CAAC,OAAO,GAAG,KAAK,UAAU,OAAO;IACrC,MAAM,IAAA,qBAAU,EAAC,+CAA+C,EAAE,EAAE,CAAC,CAAC;AACxE,CAAC,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -3,9 +3,10 @@ interface MigrationFile {
|
|
|
3
3
|
filename: string;
|
|
4
4
|
timestamp: string;
|
|
5
5
|
}
|
|
6
|
-
export declare function readAndValidateMigrationsFromDirectory(dir: string): Promise<MigrationFile[]>;
|
|
6
|
+
export declare function readAndValidateMigrationsFromDirectory(dir: string, extensions: string[]): Promise<MigrationFile[]>;
|
|
7
7
|
export declare function sortMigrationFiles(migrationFiles: MigrationFile[]): MigrationFile[];
|
|
8
8
|
export declare function getMigrationsToExecute(migrationFiles: MigrationFile[], executedMigrations: {
|
|
9
9
|
timestamp: string | null;
|
|
10
10
|
}[]): MigrationFile[];
|
|
11
|
+
export declare function initWithLock(migrationDir: string, project: string): Promise<void>;
|
|
11
12
|
export {};
|
package/dist/index.js
CHANGED
|
@@ -25,8 +25,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
25
25
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
26
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
27
|
};
|
|
28
|
+
var _a;
|
|
28
29
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.getMigrationsToExecute = exports.sortMigrationFiles = exports.readAndValidateMigrationsFromDirectory = exports.init = void 0;
|
|
30
|
+
exports.initWithLock = exports.getMigrationsToExecute = exports.sortMigrationFiles = exports.readAndValidateMigrationsFromDirectory = exports.init = void 0;
|
|
30
31
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
31
32
|
const path_1 = __importDefault(require("path"));
|
|
32
33
|
const namedLocks = __importStar(require("@prairielearn/named-locks"));
|
|
@@ -37,7 +38,16 @@ const sql = sqldb.loadSqlEquiv(__filename);
|
|
|
37
38
|
async function init(migrationDir, project) {
|
|
38
39
|
const lockName = 'migrations';
|
|
39
40
|
logger_1.logger.verbose(`Waiting for lock ${lockName}`);
|
|
40
|
-
await namedLocks.doWithLock(lockName, {
|
|
41
|
+
await namedLocks.doWithLock(lockName, {
|
|
42
|
+
// Migrations *might* take a long time to run, so we'll enable automatic
|
|
43
|
+
// lock renewal so that our lock doesn't get killed by the Postgres
|
|
44
|
+
// idle session timeout.
|
|
45
|
+
//
|
|
46
|
+
// That said, we should generally try to keep migrations executing as
|
|
47
|
+
// quickly as possible. A long-running migration likely means that
|
|
48
|
+
// Postgres is locking a whole table, which is unacceptable in production.
|
|
49
|
+
autoRenew: true,
|
|
50
|
+
}, async () => {
|
|
41
51
|
logger_1.logger.verbose(`Acquired lock ${lockName}`);
|
|
42
52
|
await initWithLock(migrationDir, project);
|
|
43
53
|
});
|
|
@@ -48,9 +58,9 @@ exports.init = init;
|
|
|
48
58
|
* Timestamp prefixes will be of the form `YYYYMMDDHHMMSS`, which will have 14 digits.
|
|
49
59
|
* If this code is still around in the year 10000... good luck.
|
|
50
60
|
*/
|
|
51
|
-
const MIGRATION_FILENAME_REGEX = /^([0-9]{14})_.+\.
|
|
52
|
-
async function readAndValidateMigrationsFromDirectory(dir) {
|
|
53
|
-
const migrationFiles = (await fs_extra_1.default.readdir(dir)).filter((m) => m.endsWith(
|
|
61
|
+
const MIGRATION_FILENAME_REGEX = /^([0-9]{14})_.+\.[a-z]+$/;
|
|
62
|
+
async function readAndValidateMigrationsFromDirectory(dir, extensions) {
|
|
63
|
+
const migrationFiles = (await fs_extra_1.default.readdir(dir)).filter((m) => extensions.some((e) => m.endsWith(e)));
|
|
54
64
|
const migrations = migrationFiles.map((mf) => {
|
|
55
65
|
const match = mf.match(MIGRATION_FILENAME_REGEX);
|
|
56
66
|
if (!match) {
|
|
@@ -67,7 +77,7 @@ async function readAndValidateMigrationsFromDirectory(dir) {
|
|
|
67
77
|
});
|
|
68
78
|
// First pass: validate that all migrations have a unique timestamp prefix.
|
|
69
79
|
// This will avoid data loss and conflicts in unexpected scenarios.
|
|
70
|
-
|
|
80
|
+
const seenTimestamps = new Set();
|
|
71
81
|
for (const migration of migrations) {
|
|
72
82
|
const { filename, timestamp } = migration;
|
|
73
83
|
if (timestamp !== null) {
|
|
@@ -125,7 +135,12 @@ async function initWithLock(migrationDir, project) {
|
|
|
125
135
|
}
|
|
126
136
|
}
|
|
127
137
|
let allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });
|
|
128
|
-
const migrationFiles = await readAndValidateMigrationsFromDirectory(migrationDir
|
|
138
|
+
const migrationFiles = await readAndValidateMigrationsFromDirectory(migrationDir, [
|
|
139
|
+
'.sql',
|
|
140
|
+
'.js',
|
|
141
|
+
'.ts',
|
|
142
|
+
'.mjs',
|
|
143
|
+
]);
|
|
129
144
|
// Validation: if we not all previously-executed migrations have timestamps,
|
|
130
145
|
// prompt the user to deploy an earlier version that includes both indexes
|
|
131
146
|
// and timestamps.
|
|
@@ -153,15 +168,24 @@ async function initWithLock(migrationDir, project) {
|
|
|
153
168
|
else {
|
|
154
169
|
logger_1.logger.info(`Running migration ${filename}`);
|
|
155
170
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
171
|
+
const migrationPath = path_1.default.join(migrationDir, filename);
|
|
172
|
+
if (filename.endsWith('.sql')) {
|
|
173
|
+
const migrationSql = await fs_extra_1.default.readFile(migrationPath, 'utf8');
|
|
174
|
+
try {
|
|
175
|
+
await sqldb.queryAsync(migrationSql, {});
|
|
176
|
+
}
|
|
177
|
+
catch (err) {
|
|
178
|
+
error.addData(err, { sqlFile: filename });
|
|
179
|
+
throw err;
|
|
180
|
+
}
|
|
161
181
|
}
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
182
|
+
else {
|
|
183
|
+
const migrationModule = await (_a = migrationPath, Promise.resolve().then(() => __importStar(require(_a))));
|
|
184
|
+
const implementation = migrationModule.default;
|
|
185
|
+
if (typeof implementation !== 'function') {
|
|
186
|
+
throw new Error(`Migration ${filename} does not export a default function`);
|
|
187
|
+
}
|
|
188
|
+
await implementation();
|
|
165
189
|
}
|
|
166
190
|
// Record the migration.
|
|
167
191
|
await sqldb.queryAsync(sql.insert_migration, {
|
|
@@ -171,4 +195,5 @@ async function initWithLock(migrationDir, project) {
|
|
|
171
195
|
});
|
|
172
196
|
}
|
|
173
197
|
}
|
|
198
|
+
exports.initWithLock = initWithLock;
|
|
174
199
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,wDAA0B;AAC1B,gDAAwB;AAExB,sEAAwD;AACxD,iDAA8C;AAC9C,8DAAgD;AAChD,2DAA6C;AAE7C,MAAM,GAAG,GAAG,KAAK,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;AAEpC,KAAK,UAAU,IAAI,CAAC,YAAoB,EAAE,OAAe;IAC9D,MAAM,QAAQ,GAAG,YAAY,CAAC;IAC9B,eAAM,CAAC,OAAO,CAAC,oBAAoB,QAAQ,EAAE,CAAC,CAAC;IAC/C,MAAM,UAAU,CAAC,UAAU,CACzB,QAAQ,EACR;QACE,wEAAwE;QACxE,mEAAmE;QACnE,wBAAwB;QACxB,EAAE;QACF,qEAAqE;QACrE,kEAAkE;QAClE,0EAA0E;QAC1E,SAAS,EAAE,IAAI;KAChB,EACD,KAAK,IAAI,EAAE;QACT,eAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;QAC5C,MAAM,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC,CACF,CAAC;IACF,eAAM,CAAC,OAAO,CAAC,iBAAiB,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AArBD,oBAqBC;AAED;;;GAGG;AACH,MAAM,wBAAwB,GAAG,0BAA0B,CAAC;AAOrD,KAAK,UAAU,sCAAsC,CAC1D,GAAW,EACX,UAAoB;IAEpB,MAAM,cAAc,GAAG,CAAC,MAAM,kBAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1D,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACtC,CAAC;IAEF,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC3C,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QAEjD,IAAI,CAAC,KAAK,EAAE;YACV,MAAM,IAAI,KAAK,CAAC,+BAA+B,EAAE,EAAE,CAAC,CAAC;SACtD;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC;QAEnC,IAAI,SAAS,KAAK,IAAI,EAAE;YACtB,MAAM,IAAI,KAAK,CAAC,aAAa,EAAE,4BAA4B,CAAC,CAAC;SAC9D;QAED,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,SAAS;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,2EAA2E;IAC3E,mEAAmE;IACnE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAE,CAAC;IACjC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE;QAClC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,SAAS,CAAC;QAE1C,IAAI,SAAS,KAAK,IAAI,EAAE;YACtB,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE;gBACjC,MAAM,IAAI,KAAK,CAAC,kCAAkC,SAAS,KAAK,QAAQ,GAAG,CAAC,CAAC;aAC9E;YACD,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;SAC/B;KACF;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AA1CD,wFA0CC;AAED,SAAgB,kBAAkB,CAAC,cAA+B;IAChE,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAClC,OAAO,CAAC,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC;AAJD,gDAIC;AAED,SAAgB,sBAAsB,CACpC,cAA+B,EAC/B,kBAAkD;IAElD,qDAAqD;IACrD,IAAI,kBAAkB,CAAC,MAAM,KAAK,CAAC,EAAE;QACnC,OAAO,cAAc,CAAC;KACvB;IAED,MAAM,2BAA2B,GAAG,IAAI,GAAG,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;IACxF,OAAO,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;AACrF,CAAC;AAXD,wDAWC;AAEM,KAAK,UAAU,YAAY,CAAC,YAAoB,EAAE,OAAe;IACtE,eAAM,CAAC,OAAO,CAAC,8BAA8B,CAAC,CAAC;IAE/C,wCAAwC;IACxC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,uBAAuB,EAAE,EAAE,CAAC,CAAC;IAExD,6DAA6D;IAC7D,IAAI;QACF,MAAM,KAAK,CAAC,UAAU,CAAC,iCAAiC,EAAE,EAAE,CAAC,CAAC;KAC/D;IAAC,OAAO,GAAQ,EAAE;QACjB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE;YACxC,eAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;YACzC,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;SACrD;aAAM;YACL,MAAM,GAAG,CAAC;SACX;KACF;IACD,IAAI;QACF,MAAM,KAAK,CAAC,UAAU,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAC;KACjE;IAAC,OAAO,GAAQ,EAAE;QACjB,IAAI,GAAG,CAAC,OAAO,KAAK,oBAAoB,EAAE;YACxC,eAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;SACtD;aAAM;YACL,MAAM,GAAG,CAAC;SACX;KACF;IAED,IAAI,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAE5E,MAAM,cAAc,GAAG,MAAM,sCAAsC,CAAC,YAAY,EAAE;QAChF,MAAM;QACN,KAAK;QACL,KAAK;QACL,MAAM;KACP,CAAC,CAAC;IAEH,4EAA4E;IAC5E,0EAA0E;IAC1E,kBAAkB;IAClB,MAAM,2BAA2B,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACnF,IAAI,2BAA2B,CAAC,MAAM,GAAG,CAAC,EAAE;QAC1C,MAAM,IAAI,KAAK,CACb;YACE,kDAAkD;YAClD,2BAA2B,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC;YACzD,kEAAkE;YAClE,qCAAqC;YACrC,0EAA0E;SAC3E,CAAC,IAAI,CAAC,IAAI,CAAC,CACb,CAAC;KACH;IAED,oDAAoD;IACpD,aAAa,GAAG,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAExE,iDAAiD;IACjD,MAAM,oBAAoB,GAAG,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAEhE,kDAAkD;IAClD,MAAM,mBAAmB,GAAG,sBAAsB,CAAC,oBAAoB,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;IAE7F,KAAK,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,mBAAmB,EAAE;QACzD,IAAI,aAAa,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE;YACnC,iEAAiE;YACjE,eAAM,CAAC,OAAO,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;SACjD;aAAM;YACL,eAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;SAC9C;QAED,MAAM,aAAa,GAAG,cAAI,CAAC,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAC;QACxD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;YAC7B,MAAM,YAAY,GAAG,MAAM,kBAAE,CAAC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;YAC9D,IAAI;gBACF,MAAM,KAAK,CAAC,UAAU,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;aAC1C;YAAC,OAAO,GAAG,EAAE;gBACZ,KAAK,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC1C,MAAM,GAAG,CAAC;aACX;SACF;aAAM;YACL,MAAM,eAAe,GAAG,YAAa,aAAa,0DAAC,CAAC;YACpD,MAAM,cAAc,GAAG,eAAe,CAAC,OAAO,CAAC;YAC/C,IAAI,OAAO,cAAc,KAAK,UAAU,EAAE;gBACxC,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,qCAAqC,CAAC,CAAC;aAC7E;YACD,MAAM,cAAc,EAAE,CAAC;SACxB;QAED,wBAAwB;QACxB,MAAM,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,gBAAgB,EAAE;YAC3C,QAAQ,EAAE,QAAQ;YAClB,SAAS;YACT,OAAO;SACR,CAAC,CAAC;KACJ;AACH,CAAC;AA/FD,oCA+FC"}
|
package/dist/index.test.js
CHANGED
|
@@ -32,6 +32,7 @@ const path_1 = __importDefault(require("path"));
|
|
|
32
32
|
const tmp_promise_1 = __importDefault(require("tmp-promise"));
|
|
33
33
|
const fs_extra_1 = __importDefault(require("fs-extra"));
|
|
34
34
|
const index_1 = require("./index");
|
|
35
|
+
const postgres_1 = require("@prairielearn/postgres");
|
|
35
36
|
chai_1.default.use(chai_as_promised_1.default);
|
|
36
37
|
async function withMigrationFiles(files, fn) {
|
|
37
38
|
await tmp_promise_1.default.withDir(async function (tmpDir) {
|
|
@@ -45,12 +46,12 @@ describe('migrations', () => {
|
|
|
45
46
|
describe('readAndValidateMigrationsFromDirectory', () => {
|
|
46
47
|
it('handles migrations without a timestamp', async () => {
|
|
47
48
|
await withMigrationFiles(['001_testing.sql'], async (tmpDir) => {
|
|
48
|
-
await chai_1.assert.isRejected((0, index_1.readAndValidateMigrationsFromDirectory)(tmpDir), 'Invalid migration filename: 001_testing.sql');
|
|
49
|
+
await chai_1.assert.isRejected((0, index_1.readAndValidateMigrationsFromDirectory)(tmpDir, ['.sql']), 'Invalid migration filename: 001_testing.sql');
|
|
49
50
|
});
|
|
50
51
|
});
|
|
51
52
|
it('handles duplicate timestamps', async () => {
|
|
52
53
|
await withMigrationFiles(['20220101010101_testing.sql', '20220101010101_testing_again.sql'], async (tmpDir) => {
|
|
53
|
-
await chai_1.assert.isRejected((0, index_1.readAndValidateMigrationsFromDirectory)(tmpDir), 'Duplicate migration timestamp');
|
|
54
|
+
await chai_1.assert.isRejected((0, index_1.readAndValidateMigrationsFromDirectory)(tmpDir, ['.sql']), 'Duplicate migration timestamp');
|
|
54
55
|
});
|
|
55
56
|
});
|
|
56
57
|
});
|
|
@@ -123,5 +124,25 @@ describe('migrations', () => {
|
|
|
123
124
|
]);
|
|
124
125
|
});
|
|
125
126
|
});
|
|
127
|
+
describe('initWithLock', () => {
|
|
128
|
+
const postgresTestUtils = (0, postgres_1.makePostgresTestUtils)({
|
|
129
|
+
database: 'prairielearn_migrations',
|
|
130
|
+
});
|
|
131
|
+
before(async () => {
|
|
132
|
+
await postgresTestUtils.createDatabase();
|
|
133
|
+
});
|
|
134
|
+
after(async () => {
|
|
135
|
+
await postgresTestUtils.dropDatabase();
|
|
136
|
+
});
|
|
137
|
+
it('runs both SQL and JavaScript migrations', async () => {
|
|
138
|
+
const migrationDir = path_1.default.join(__dirname, 'fixtures');
|
|
139
|
+
await (0, index_1.initWithLock)(migrationDir, 'prairielearn_migrations');
|
|
140
|
+
// If both migrations ran successfully, there should be a single user
|
|
141
|
+
// in the database.
|
|
142
|
+
const users = await (0, postgres_1.queryAsync)('SELECT * FROM users', {});
|
|
143
|
+
chai_1.assert.lengthOf(users.rows, 1);
|
|
144
|
+
chai_1.assert.equal(users.rows[0].name, 'Test User');
|
|
145
|
+
});
|
|
146
|
+
});
|
|
126
147
|
});
|
|
127
148
|
//# sourceMappingURL=index.test.js.map
|
package/dist/index.test.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAoC;AACpC,wEAA8C;AAC9C,gDAAwB;AACxB,8DAA8B;AAC9B,wDAA0B;AAE1B,
|
|
1
|
+
{"version":3,"file":"index.test.js","sourceRoot":"","sources":["../src/index.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,6CAAoC;AACpC,wEAA8C;AAC9C,gDAAwB;AACxB,8DAA8B;AAC9B,wDAA0B;AAE1B,mCAKiB;AACjB,qDAA2E;AAE3E,cAAI,CAAC,GAAG,CAAC,0BAAc,CAAC,CAAC;AAEzB,KAAK,UAAU,kBAAkB,CAAC,KAAe,EAAE,EAAqC;IACtF,MAAM,qBAAG,CAAC,OAAO,CACf,KAAK,WAAW,MAAM;QACpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE;YACxB,MAAM,kBAAE,CAAC,SAAS,CAAC,cAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;SACtD;QACD,MAAM,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACxB,CAAC,EACD,EAAE,aAAa,EAAE,IAAI,EAAE,CACxB,CAAC;AACJ,CAAC;AAED,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,QAAQ,CAAC,wCAAwC,EAAE,GAAG,EAAE;QACtD,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,kBAAkB,CAAC,CAAC,iBAAiB,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;gBAC7D,MAAM,aAAM,CAAC,UAAU,CACrB,IAAA,8CAAsC,EAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EACxD,6CAA6C,CAC9C,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,kBAAkB,CACtB,CAAC,4BAA4B,EAAE,kCAAkC,CAAC,EAClE,KAAK,EAAE,MAAM,EAAE,EAAE;gBACf,MAAM,aAAM,CAAC,UAAU,CACrB,IAAA,8CAAsC,EAAC,MAAM,EAAE,CAAC,MAAM,CAAC,CAAC,EACxD,+BAA+B,CAChC,CAAC;YACJ,CAAC,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,oBAAoB,EAAE,GAAG,EAAE;YAC5B,aAAM,CAAC,SAAS,CACd,IAAA,0BAAkB,EAAC;gBACjB;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC,EACF;gBACE;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,wBAAwB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;YACpD,MAAM,cAAc,GAAG;gBACrB;oBACE,QAAQ,EAAE,iBAAiB;oBAC3B,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,8BAAsB,EAAC,cAAc,EAAE,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/E,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,4DAA4D,EAAE,GAAG,EAAE;YACpE,MAAM,cAAc,GAAG;gBACrB;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,QAAQ,EAAE,8BAA8B;oBACxC,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,MAAM,kBAAkB,GAAG;gBACzB;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;gBACD;oBACE,SAAS,EAAE,gBAAgB;iBAC5B;aACF,CAAC;YACF,aAAM,CAAC,SAAS,CAAC,IAAA,8BAAsB,EAAC,cAAc,EAAE,kBAAkB,CAAC,EAAE;gBAC3E,EAAE,SAAS,EAAE,gBAAgB,EAAE,QAAQ,EAAE,8BAA8B,EAAE;aAC1E,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,MAAM,iBAAiB,GAAG,IAAA,gCAAqB,EAAC;YAC9C,QAAQ,EAAE,yBAAyB;SACpC,CAAC,CAAC;QAEH,MAAM,CAAC,KAAK,IAAI,EAAE;YAChB,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC;QAC3C,CAAC,CAAC,CAAC;QAEH,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,MAAM,iBAAiB,CAAC,YAAY,EAAE,CAAC;QACzC,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,GAAG,cAAI,CAAC,IAAI,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;YACtD,MAAM,IAAA,oBAAY,EAAC,YAAY,EAAE,yBAAyB,CAAC,CAAC;YAE5D,qEAAqE;YACrE,mBAAmB;YACnB,MAAM,KAAK,GAAG,MAAM,IAAA,qBAAU,EAAC,qBAAqB,EAAE,EAAE,CAAC,CAAC;YAC1D,aAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC/B,aAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@prairielearn/migrations",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"main": "./dist/index.js",
|
|
5
|
+
"repository": {
|
|
6
|
+
"type": "git",
|
|
7
|
+
"url": "https://github.com/PrairieLearn/PrairieLearn.git",
|
|
8
|
+
"directory": "packages/migrations"
|
|
9
|
+
},
|
|
5
10
|
"scripts": {
|
|
6
11
|
"build": "tsc && copyfiles -u 1 \"./src/**/*.sql\" dist",
|
|
7
12
|
"dev": "tsc --watch --preserveWatchOutput",
|
|
@@ -11,7 +16,7 @@
|
|
|
11
16
|
"@prairielearn/tsconfig": "*",
|
|
12
17
|
"@types/fs-extra": "^11.0.1",
|
|
13
18
|
"@types/mocha": "^10.0.1",
|
|
14
|
-
"@types/node": "^18.
|
|
19
|
+
"@types/node": "^18.15.11",
|
|
15
20
|
"copyfiles": "^2.4.1",
|
|
16
21
|
"mocha": "^10.2.0",
|
|
17
22
|
"ts-node": "^10.9.1",
|
|
@@ -19,8 +24,8 @@
|
|
|
19
24
|
},
|
|
20
25
|
"dependencies": {
|
|
21
26
|
"@prairielearn/error": "^1.0.0",
|
|
22
|
-
"@prairielearn/named-locks": "^1.
|
|
27
|
+
"@prairielearn/named-locks": "^1.2.0",
|
|
23
28
|
"@prairielearn/postgres": "^1.2.0",
|
|
24
|
-
"fs-extra": "^11.1.
|
|
29
|
+
"fs-extra": "^11.1.1"
|
|
25
30
|
}
|
|
26
31
|
}
|
package/src/index.test.ts
CHANGED
|
@@ -8,7 +8,9 @@ import {
|
|
|
8
8
|
readAndValidateMigrationsFromDirectory,
|
|
9
9
|
sortMigrationFiles,
|
|
10
10
|
getMigrationsToExecute,
|
|
11
|
+
initWithLock,
|
|
11
12
|
} from './index';
|
|
13
|
+
import { makePostgresTestUtils, queryAsync } from '@prairielearn/postgres';
|
|
12
14
|
|
|
13
15
|
chai.use(chaiAsPromised);
|
|
14
16
|
|
|
@@ -29,7 +31,7 @@ describe('migrations', () => {
|
|
|
29
31
|
it('handles migrations without a timestamp', async () => {
|
|
30
32
|
await withMigrationFiles(['001_testing.sql'], async (tmpDir) => {
|
|
31
33
|
await assert.isRejected(
|
|
32
|
-
readAndValidateMigrationsFromDirectory(tmpDir),
|
|
34
|
+
readAndValidateMigrationsFromDirectory(tmpDir, ['.sql']),
|
|
33
35
|
'Invalid migration filename: 001_testing.sql'
|
|
34
36
|
);
|
|
35
37
|
});
|
|
@@ -40,7 +42,7 @@ describe('migrations', () => {
|
|
|
40
42
|
['20220101010101_testing.sql', '20220101010101_testing_again.sql'],
|
|
41
43
|
async (tmpDir) => {
|
|
42
44
|
await assert.isRejected(
|
|
43
|
-
readAndValidateMigrationsFromDirectory(tmpDir),
|
|
45
|
+
readAndValidateMigrationsFromDirectory(tmpDir, ['.sql']),
|
|
44
46
|
'Duplicate migration timestamp'
|
|
45
47
|
);
|
|
46
48
|
}
|
|
@@ -122,4 +124,29 @@ describe('migrations', () => {
|
|
|
122
124
|
]);
|
|
123
125
|
});
|
|
124
126
|
});
|
|
127
|
+
|
|
128
|
+
describe('initWithLock', () => {
|
|
129
|
+
const postgresTestUtils = makePostgresTestUtils({
|
|
130
|
+
database: 'prairielearn_migrations',
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
before(async () => {
|
|
134
|
+
await postgresTestUtils.createDatabase();
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
after(async () => {
|
|
138
|
+
await postgresTestUtils.dropDatabase();
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it('runs both SQL and JavaScript migrations', async () => {
|
|
142
|
+
const migrationDir = path.join(__dirname, 'fixtures');
|
|
143
|
+
await initWithLock(migrationDir, 'prairielearn_migrations');
|
|
144
|
+
|
|
145
|
+
// If both migrations ran successfully, there should be a single user
|
|
146
|
+
// in the database.
|
|
147
|
+
const users = await queryAsync('SELECT * FROM users', {});
|
|
148
|
+
assert.lengthOf(users.rows, 1);
|
|
149
|
+
assert.equal(users.rows[0].name, 'Test User');
|
|
150
|
+
});
|
|
151
|
+
});
|
|
125
152
|
});
|
package/src/index.ts
CHANGED
|
@@ -11,10 +11,23 @@ const sql = sqldb.loadSqlEquiv(__filename);
|
|
|
11
11
|
export async function init(migrationDir: string, project: string) {
|
|
12
12
|
const lockName = 'migrations';
|
|
13
13
|
logger.verbose(`Waiting for lock ${lockName}`);
|
|
14
|
-
await namedLocks.doWithLock(
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
14
|
+
await namedLocks.doWithLock(
|
|
15
|
+
lockName,
|
|
16
|
+
{
|
|
17
|
+
// Migrations *might* take a long time to run, so we'll enable automatic
|
|
18
|
+
// lock renewal so that our lock doesn't get killed by the Postgres
|
|
19
|
+
// idle session timeout.
|
|
20
|
+
//
|
|
21
|
+
// That said, we should generally try to keep migrations executing as
|
|
22
|
+
// quickly as possible. A long-running migration likely means that
|
|
23
|
+
// Postgres is locking a whole table, which is unacceptable in production.
|
|
24
|
+
autoRenew: true,
|
|
25
|
+
},
|
|
26
|
+
async () => {
|
|
27
|
+
logger.verbose(`Acquired lock ${lockName}`);
|
|
28
|
+
await initWithLock(migrationDir, project);
|
|
29
|
+
}
|
|
30
|
+
);
|
|
18
31
|
logger.verbose(`Released lock ${lockName}`);
|
|
19
32
|
}
|
|
20
33
|
|
|
@@ -22,7 +35,7 @@ export async function init(migrationDir: string, project: string) {
|
|
|
22
35
|
* Timestamp prefixes will be of the form `YYYYMMDDHHMMSS`, which will have 14 digits.
|
|
23
36
|
* If this code is still around in the year 10000... good luck.
|
|
24
37
|
*/
|
|
25
|
-
const MIGRATION_FILENAME_REGEX = /^([0-9]{14})_.+\.
|
|
38
|
+
const MIGRATION_FILENAME_REGEX = /^([0-9]{14})_.+\.[a-z]+$/;
|
|
26
39
|
|
|
27
40
|
interface MigrationFile {
|
|
28
41
|
filename: string;
|
|
@@ -30,9 +43,12 @@ interface MigrationFile {
|
|
|
30
43
|
}
|
|
31
44
|
|
|
32
45
|
export async function readAndValidateMigrationsFromDirectory(
|
|
33
|
-
dir: string
|
|
46
|
+
dir: string,
|
|
47
|
+
extensions: string[]
|
|
34
48
|
): Promise<MigrationFile[]> {
|
|
35
|
-
const migrationFiles = (await fs.readdir(dir)).filter((m) =>
|
|
49
|
+
const migrationFiles = (await fs.readdir(dir)).filter((m) =>
|
|
50
|
+
extensions.some((e) => m.endsWith(e))
|
|
51
|
+
);
|
|
36
52
|
|
|
37
53
|
const migrations = migrationFiles.map((mf) => {
|
|
38
54
|
const match = mf.match(MIGRATION_FILENAME_REGEX);
|
|
@@ -55,7 +71,7 @@ export async function readAndValidateMigrationsFromDirectory(
|
|
|
55
71
|
|
|
56
72
|
// First pass: validate that all migrations have a unique timestamp prefix.
|
|
57
73
|
// This will avoid data loss and conflicts in unexpected scenarios.
|
|
58
|
-
|
|
74
|
+
const seenTimestamps = new Set();
|
|
59
75
|
for (const migration of migrations) {
|
|
60
76
|
const { filename, timestamp } = migration;
|
|
61
77
|
|
|
@@ -89,7 +105,7 @@ export function getMigrationsToExecute(
|
|
|
89
105
|
return migrationFiles.filter((m) => !executedMigrationTimestamps.has(m.timestamp));
|
|
90
106
|
}
|
|
91
107
|
|
|
92
|
-
async function initWithLock(migrationDir: string, project: string) {
|
|
108
|
+
export async function initWithLock(migrationDir: string, project: string) {
|
|
93
109
|
logger.verbose('Starting DB schema migration');
|
|
94
110
|
|
|
95
111
|
// Create the migrations table if needed
|
|
@@ -119,7 +135,12 @@ async function initWithLock(migrationDir: string, project: string) {
|
|
|
119
135
|
|
|
120
136
|
let allMigrations = await sqldb.queryAsync(sql.get_migrations, { project });
|
|
121
137
|
|
|
122
|
-
const migrationFiles = await readAndValidateMigrationsFromDirectory(migrationDir
|
|
138
|
+
const migrationFiles = await readAndValidateMigrationsFromDirectory(migrationDir, [
|
|
139
|
+
'.sql',
|
|
140
|
+
'.js',
|
|
141
|
+
'.ts',
|
|
142
|
+
'.mjs',
|
|
143
|
+
]);
|
|
123
144
|
|
|
124
145
|
// Validation: if we not all previously-executed migrations have timestamps,
|
|
125
146
|
// prompt the user to deploy an earlier version that includes both indexes
|
|
@@ -154,15 +175,22 @@ async function initWithLock(migrationDir: string, project: string) {
|
|
|
154
175
|
logger.info(`Running migration ${filename}`);
|
|
155
176
|
}
|
|
156
177
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
178
|
+
const migrationPath = path.join(migrationDir, filename);
|
|
179
|
+
if (filename.endsWith('.sql')) {
|
|
180
|
+
const migrationSql = await fs.readFile(migrationPath, 'utf8');
|
|
181
|
+
try {
|
|
182
|
+
await sqldb.queryAsync(migrationSql, {});
|
|
183
|
+
} catch (err) {
|
|
184
|
+
error.addData(err, { sqlFile: filename });
|
|
185
|
+
throw err;
|
|
186
|
+
}
|
|
187
|
+
} else {
|
|
188
|
+
const migrationModule = await import(migrationPath);
|
|
189
|
+
const implementation = migrationModule.default;
|
|
190
|
+
if (typeof implementation !== 'function') {
|
|
191
|
+
throw new Error(`Migration ${filename} does not export a default function`);
|
|
192
|
+
}
|
|
193
|
+
await implementation();
|
|
166
194
|
}
|
|
167
195
|
|
|
168
196
|
// Record the migration.
|