@apso/cli 0.43.3 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -11
- package/dist/commands/deploy.js +1 -1
- package/dist/commands/generate.js +22 -12
- package/dist/commands/migrate/generate.d.ts +3 -1
- package/dist/commands/migrate/generate.js +42 -8
- package/dist/commands/migrate/plan.d.ts +12 -0
- package/dist/commands/migrate/plan.js +88 -0
- package/dist/commands/migrate/sandbox.d.ts +11 -0
- package/dist/commands/migrate/sandbox.js +132 -0
- package/dist/commands/migrate/verify.d.ts +5 -2
- package/dist/commands/migrate/verify.js +44 -9
- package/dist/commands/migrate.d.ts +16 -3
- package/dist/commands/migrate.js +171 -96
- package/dist/lib/ddl/catalog.d.ts +21 -0
- package/dist/lib/ddl/catalog.js +58 -0
- package/dist/lib/ddl/change.d.ts +19 -0
- package/dist/lib/ddl/change.js +393 -0
- package/dist/lib/ddl/index.d.ts +80 -0
- package/dist/lib/ddl/index.js +309 -0
- package/dist/lib/ddl/migration-file.d.ts +42 -0
- package/dist/lib/ddl/migration-file.js +222 -0
- package/dist/lib/generators/go.js +3 -2
- package/dist/lib/generators/python.js +3 -2
- package/dist/lib/plan/baseline.d.ts +22 -0
- package/dist/lib/plan/baseline.js +100 -0
- package/dist/lib/plan/index.d.ts +52 -0
- package/dist/lib/plan/index.js +272 -0
- package/dist/lib/plan/prompts.d.ts +18 -0
- package/dist/lib/plan/prompts.js +77 -0
- package/dist/lib/utils/file-system.d.ts +8 -0
- package/dist/lib/utils/file-system.js +31 -1
- package/dist/lib/verify/index.d.ts +27 -9
- package/dist/lib/verify/index.js +249 -21
- package/dist/lib/verify/review.d.ts +14 -0
- package/dist/lib/verify/review.js +98 -0
- package/npm-shrinkwrap.json +2 -2
- package/oclif.manifest.json +176 -23
- package/package.json +1 -1
package/dist/lib/verify/index.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
/**
|
|
3
3
|
* Pre-deploy checks for a generated service: generate the migration from the
|
|
4
|
-
* deployed schema, run it and check for drift (
|
|
5
|
-
*
|
|
4
|
+
* deployed schema, run it and check for drift (on PGlite), and smoke test the
|
|
5
|
+
* endpoints. The CLI and the in-browser editor (which runs
|
|
6
6
|
* the CLI in a WebContainer) share these steps.
|
|
7
7
|
*/
|
|
8
8
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
9
|
-
exports.report = exports.cliBinPath = exports.readDotEnv = exports.smokeTest = exports.entityRoute = exports.verifyMigration = exports.supportsPglite = exports.
|
|
9
|
+
exports.report = exports.cliBinPath = exports.readDotEnv = exports.smokeTest = exports.entityRoute = exports.verifyMigration = exports.supportsPglite = exports.unaccepted = exports.dataLoss = exports.generateMigration = exports.buildBaseline = exports.pgliteEnv = exports.run = void 0;
|
|
10
10
|
const tslib_1 = require("tslib");
|
|
11
11
|
const child_process_1 = require("child_process");
|
|
12
12
|
const fs = tslib_1.__importStar(require("fs"));
|
|
@@ -15,8 +15,13 @@ const net = tslib_1.__importStar(require("net"));
|
|
|
15
15
|
const os = tslib_1.__importStar(require("os"));
|
|
16
16
|
const path = tslib_1.__importStar(require("path"));
|
|
17
17
|
const pluralize_1 = tslib_1.__importDefault(require("pluralize"));
|
|
18
|
+
const apsorc_parser_1 = require("../apsorc-parser");
|
|
19
|
+
const ddl_1 = require("../ddl");
|
|
20
|
+
const catalog_1 = require("../ddl/catalog");
|
|
21
|
+
const migration_file_1 = require("../ddl/migration-file");
|
|
18
22
|
const casing_1 = require("../utils/casing");
|
|
19
23
|
const crud_plan_1 = require("./crud-plan");
|
|
24
|
+
const review_1 = require("./review");
|
|
20
25
|
// eslint-disable-next-line no-control-regex
|
|
21
26
|
const plain = (s) => s.replace(/\u001B\[[\d;]*m/g, "");
|
|
22
27
|
const tail = (s, n = 4000) => {
|
|
@@ -85,19 +90,6 @@ async function buildBaseline(projectDir, baselineApsorc, dataDir, cliBin) {
|
|
|
85
90
|
}
|
|
86
91
|
}
|
|
87
92
|
exports.buildBaseline = buildBaseline;
|
|
88
|
-
/**
|
|
89
|
-
* Why `apso migrate` can't run for this project's language, or undefined.
|
|
90
|
-
* ponytail: Python (Alembic) and Go (goose) have no local scratch database to
|
|
91
|
-
* build the baseline on; add when they get one like PGlite for TypeScript.
|
|
92
|
-
*/
|
|
93
|
-
const migrateUnsupported = (language = "typescript") => {
|
|
94
|
-
if (language === "python")
|
|
95
|
-
return "apso migrate supports TypeScript services only for now. This is a Python service: use Alembic directly (alembic revision --autogenerate, alembic upgrade head, alembic check).";
|
|
96
|
-
if (language === "go")
|
|
97
|
-
return "apso migrate supports TypeScript services only for now. This is a Go service: write goose migrations in migrations/ and run them with `go run cmd/migrate/main.go up`.";
|
|
98
|
-
return undefined;
|
|
99
|
-
};
|
|
100
|
-
exports.migrateUnsupported = migrateUnsupported;
|
|
101
93
|
/** Generate a migration from the deployed schema (baselineApsorc) to the project's entities. */
|
|
102
94
|
async function generateMigration(projectDir, baselineApsorc, name, cliBin) {
|
|
103
95
|
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-pglite-"));
|
|
@@ -129,6 +121,9 @@ async function generateMigration(projectDir, baselineApsorc, name, cliBin) {
|
|
|
129
121
|
needed: true,
|
|
130
122
|
file: file && path.join("src", "migrations", file),
|
|
131
123
|
steps,
|
|
124
|
+
findings: file
|
|
125
|
+
? (0, review_1.reviewMigration)(fs.readFileSync(path.join(migrationsDir, file), "utf8"))
|
|
126
|
+
: undefined,
|
|
132
127
|
};
|
|
133
128
|
}
|
|
134
129
|
finally {
|
|
@@ -148,6 +143,96 @@ ds.initialize()
|
|
|
148
143
|
.then((ran) => { console.log("ran " + ran.map((m) => m.name).join(", ")); return ds.destroy(); })
|
|
149
144
|
.catch((e) => { console.error(e && e.message ? e.message : e); process.exit(1); });
|
|
150
145
|
`;
|
|
146
|
+
// Row and non-null value counts per table and column of the PGlite database
|
|
147
|
+
// at PGLITE_DATA_DIR, after running the SQL file in argv[1] if given, plus each
|
|
148
|
+
// column's distinct values (as text) so data moved to another table is found.
|
|
149
|
+
const COUNT_ROWS = `
|
|
150
|
+
const fs = require("fs");
|
|
151
|
+
const { PGlite } = require("@electric-sql/pglite");
|
|
152
|
+
(async () => {
|
|
153
|
+
const db = new PGlite(process.env.PGLITE_DATA_DIR);
|
|
154
|
+
if (process.argv[1]) await db.exec(fs.readFileSync(process.argv[1], "utf8"));
|
|
155
|
+
const cols = (await db.query("SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name <> 'migrations'")).rows;
|
|
156
|
+
const out = {};
|
|
157
|
+
for (const { table_name: t, column_name: c } of cols) {
|
|
158
|
+
const q = '"' + c + '"';
|
|
159
|
+
const r = (await db.query("SELECT count(*)::int AS rows, count(" + q + ")::int AS vals, (SELECT array_agg(v) FROM (SELECT DISTINCT " + q + "::text AS v FROM \\"" + t + "\\" WHERE " + q + " IS NOT NULL LIMIT 1000) d) AS dv FROM \\"" + t + "\\"")).rows[0];
|
|
160
|
+
out[t] = out[t] || { rows: r.rows, values: {}, distinct: {} };
|
|
161
|
+
out[t].values[c] = r.vals;
|
|
162
|
+
out[t].distinct[c] = r.dv || [];
|
|
163
|
+
}
|
|
164
|
+
console.log(JSON.stringify(out));
|
|
165
|
+
await db.close();
|
|
166
|
+
})().catch((e) => { console.error(e.message); process.exit(1); });
|
|
167
|
+
`;
|
|
168
|
+
// Assertions the user wrote for this migration: each SELECT returns rows with a
|
|
169
|
+
// boolean "ok" and a "label", the same format as the eval checks.
|
|
170
|
+
const RUN_CHECKS = `
|
|
171
|
+
const fs = require("fs");
|
|
172
|
+
const { PGlite } = require("@electric-sql/pglite");
|
|
173
|
+
(async () => {
|
|
174
|
+
const db = new PGlite(process.env.PGLITE_DATA_DIR);
|
|
175
|
+
const failed = [];
|
|
176
|
+
for (const r of await db.exec(fs.readFileSync(process.argv[1], "utf8")))
|
|
177
|
+
for (const row of r.rows) if ("ok" in row && row.ok !== true) failed.push(row.label || JSON.stringify(row));
|
|
178
|
+
await db.close();
|
|
179
|
+
if (failed.length) { console.error("Failed checks:\\n" + failed.join("\\n")); process.exit(1); }
|
|
180
|
+
})().catch((e) => { console.error(e.message); process.exit(1); });
|
|
181
|
+
`;
|
|
182
|
+
// Filled in by the database on insert, so a moved row gets new ones.
|
|
183
|
+
const GENERATED = new Set(["created_at", "updated_at"]);
|
|
184
|
+
/**
|
|
185
|
+
* Rows and values present before the migration and missing after it,
|
|
186
|
+
* following the tables and columns the migration renames. A column or table
|
|
187
|
+
* whose values all turn up in another column after the migration was moved,
|
|
188
|
+
* not lost (a field moved to a new table, a foreign key moved to a join table).
|
|
189
|
+
*/
|
|
190
|
+
function dataLoss(before, after, source) {
|
|
191
|
+
var _a, _b, _c, _d;
|
|
192
|
+
const tables = new Map([...source.matchAll(/ALTER TABLE "([^"]+)" RENAME TO "([^"]+)"/g)].map((m) => [m[1], m[2]]));
|
|
193
|
+
const columns = new Map([...source.matchAll(/ALTER TABLE "([^"]+)" RENAME COLUMN "([^"]+)" TO "([^"]+)"/g)].map((m) => [`${m[1]}.${m[2]}`, m[3]]));
|
|
194
|
+
// Only tables the migration creates count, and not their ids: small integer
|
|
195
|
+
// values (foreign keys) would otherwise match any existing id column.
|
|
196
|
+
const afterValues = Object.entries(after)
|
|
197
|
+
.filter(([t]) => !before[t] && ![...tables.values()].includes(t))
|
|
198
|
+
.flatMap(([, a]) => {
|
|
199
|
+
var _a;
|
|
200
|
+
return Object.entries((_a = a.distinct) !== null && _a !== void 0 ? _a : {})
|
|
201
|
+
.filter(([c]) => c !== "id")
|
|
202
|
+
.map(([, v]) => new Set(v));
|
|
203
|
+
});
|
|
204
|
+
const moved = (t, c) => {
|
|
205
|
+
var _a, _b;
|
|
206
|
+
const vals = (_b = (_a = before[t].distinct) === null || _a === void 0 ? void 0 : _a[c]) !== null && _b !== void 0 ? _b : [];
|
|
207
|
+
return vals.length > 0 && afterValues.some((s) => vals.every((v) => s.has(v)));
|
|
208
|
+
};
|
|
209
|
+
const lost = [];
|
|
210
|
+
for (const [t, { rows, values }] of Object.entries(before)) {
|
|
211
|
+
if (!rows)
|
|
212
|
+
continue;
|
|
213
|
+
const t2 = (_a = tables.get(t)) !== null && _a !== void 0 ? _a : t;
|
|
214
|
+
const now = after[t2];
|
|
215
|
+
if (!now) {
|
|
216
|
+
const gone = Object.keys(values).filter((c) => values[c] > 0 && !GENERATED.has(c) && !moved(t, c));
|
|
217
|
+
if (gone.length > 0)
|
|
218
|
+
lost.push(`table ${t}: all ${rows} rows`);
|
|
219
|
+
continue;
|
|
220
|
+
}
|
|
221
|
+
if (now.rows < rows)
|
|
222
|
+
lost.push(`table ${t}: ${rows - now.rows} of ${rows} rows`);
|
|
223
|
+
for (const [c, n] of Object.entries(values)) {
|
|
224
|
+
const c2 = (_c = (_b = columns.get(`${t}.${c}`)) !== null && _b !== void 0 ? _b : columns.get(`${t2}.${c}`)) !== null && _c !== void 0 ? _c : c;
|
|
225
|
+
const kept = (_d = now.values[c2]) !== null && _d !== void 0 ? _d : 0;
|
|
226
|
+
if (n > kept && !(kept === 0 && moved(t, c)))
|
|
227
|
+
lost.push(`${t}.${c}: ${n - kept} of ${n} values`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return lost;
|
|
231
|
+
}
|
|
232
|
+
exports.dataLoss = dataLoss;
|
|
233
|
+
/** Losses the user accepted by name: a table, or table.column. */
|
|
234
|
+
const unaccepted = (lost, accept) => lost.filter((l) => !accept.some((a) => l.startsWith(`${a}:`) || l.startsWith(`table ${a}:`)));
|
|
235
|
+
exports.unaccepted = unaccepted;
|
|
151
236
|
/**
|
|
152
237
|
* Projects from the v1 template have no PGlite support in src/orm.config.ts,
|
|
153
238
|
* so the checks cannot run locally. They pass as skipped rather than block a
|
|
@@ -176,9 +261,17 @@ exports.supportsPglite = supportsPglite;
|
|
|
176
261
|
* Run a migration on a fresh copy of the deployed schema, then check that the
|
|
177
262
|
* database matches the project's entities (a second generate finds nothing).
|
|
178
263
|
*/
|
|
179
|
-
async function verifyMigration(projectDir, baselineApsorc, migrationFile, cliBin, skipBuild = false) {
|
|
264
|
+
async function verifyMigration(projectDir, baselineApsorc, migrationFile, cliBin, { skipBuild = false, seedFile, checkFile, acceptLoss = [], } = {}) {
|
|
265
|
+
if (/\.(py|sql)$/.test(migrationFile))
|
|
266
|
+
return verifySqlMigration(projectDir, baselineApsorc, migrationFile, cliBin, {
|
|
267
|
+
seedFile,
|
|
268
|
+
checkFile,
|
|
269
|
+
acceptLoss,
|
|
270
|
+
});
|
|
180
271
|
if (!supportsPglite(projectDir))
|
|
181
272
|
return NO_PGLITE;
|
|
273
|
+
const source = fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8");
|
|
274
|
+
const findings = (0, review_1.reviewMigration)(source);
|
|
182
275
|
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-pglite-"));
|
|
183
276
|
const steps = [];
|
|
184
277
|
const done = (s) => {
|
|
@@ -187,25 +280,157 @@ async function verifyMigration(projectDir, baselineApsorc, migrationFile, cliBin
|
|
|
187
280
|
};
|
|
188
281
|
try {
|
|
189
282
|
if (!done(await buildBaseline(projectDir, baselineApsorc, dataDir, cliBin)))
|
|
190
|
-
return { ok: false, steps };
|
|
283
|
+
return { ok: false, steps, findings };
|
|
284
|
+
const count = async (name, sql) => {
|
|
285
|
+
const r = await run(process.execPath, ["-e", COUNT_ROWS, ...(sql ? [sql] : [])], projectDir, (0, exports.pgliteEnv)(dataDir));
|
|
286
|
+
done(step(name, r));
|
|
287
|
+
return r.ok ? JSON.parse(r.output.trim().split("\n").pop()) : undefined;
|
|
288
|
+
};
|
|
289
|
+
// Seed rows make the migration face the data it will meet in production.
|
|
290
|
+
const before = seedFile ? await count("seed", path.resolve(seedFile)) : undefined;
|
|
291
|
+
if (seedFile && !before)
|
|
292
|
+
return { ok: false, steps, findings };
|
|
191
293
|
const compiled = path
|
|
192
294
|
.join("dist", path.relative("src", migrationFile))
|
|
193
295
|
.replace(/\.ts$/, ".js");
|
|
194
296
|
// --skip-build still builds when the migration isn't compiled yet.
|
|
195
297
|
const needsBuild = !skipBuild || !fs.existsSync(path.join(projectDir, compiled));
|
|
196
298
|
if (needsBuild && !done(step("build", await build(projectDir))))
|
|
197
|
-
return { ok: false, steps };
|
|
299
|
+
return { ok: false, steps, findings };
|
|
198
300
|
const migrate = await run(process.execPath, ["-e", RUN_ONE_MIGRATION, compiled], projectDir, (0, exports.pgliteEnv)(dataDir));
|
|
199
301
|
if (!done({ ...step("migrate", migrate), output: tail(migrate.output) }))
|
|
200
|
-
return { ok: false, steps };
|
|
302
|
+
return { ok: false, steps, findings };
|
|
303
|
+
if (before) {
|
|
304
|
+
const after = await count("count");
|
|
305
|
+
if (!after)
|
|
306
|
+
return { ok: false, steps, findings };
|
|
307
|
+
const lost = dataLoss(before, after, source);
|
|
308
|
+
const ok = (0, exports.unaccepted)(lost, acceptLoss).length === 0;
|
|
309
|
+
done({
|
|
310
|
+
name: "data",
|
|
311
|
+
ok,
|
|
312
|
+
output: lost.length > 0
|
|
313
|
+
? `The migration lost seeded data:\n${lost.join("\n")}${ok ? "" : "\nFix the migration. If data moved somewhere the check can't follow (e.g. a field split in two), prove it with --check and rerun with --accept-loss <table.column>."}`
|
|
314
|
+
: undefined,
|
|
315
|
+
});
|
|
316
|
+
if (!ok)
|
|
317
|
+
return { ok: false, steps, findings };
|
|
318
|
+
}
|
|
319
|
+
if (checkFile &&
|
|
320
|
+
!done(step("check", await run(process.execPath, ["-e", RUN_CHECKS, path.resolve(checkFile)], projectDir, (0, exports.pgliteEnv)(dataDir)))))
|
|
321
|
+
return { ok: false, steps, findings };
|
|
201
322
|
const drift = await typeorm(projectDir, ["migration:generate", "./src/migrations/drift", "--check"], (0, exports.pgliteEnv)(dataDir));
|
|
202
|
-
return { ok: done(step("drift", drift)), steps };
|
|
323
|
+
return { ok: done(step("drift", drift)), steps, findings };
|
|
203
324
|
}
|
|
204
325
|
finally {
|
|
205
326
|
fs.rmSync(dataDir, { recursive: true, force: true });
|
|
206
327
|
}
|
|
207
328
|
}
|
|
208
329
|
exports.verifyMigration = verifyMigration;
|
|
330
|
+
const ddlOf = (apsorcFile) => (0, ddl_1.schemaDdl)((0, apsorc_parser_1.parseApsorcV2)(JSON.parse(fs.readFileSync(apsorcFile).toString()))).join(";\n");
|
|
331
|
+
/**
|
|
332
|
+
* verifyMigration for an Alembic or goose migration. Python and Go services
|
|
333
|
+
* have no TypeORM, so Apso's DDL writer builds the deployed schema, the
|
|
334
|
+
* migration's SQL runs as written, and drift is a catalog diff against a
|
|
335
|
+
* fresh build of the new .apsorc. down then runs and must give back the
|
|
336
|
+
* deployed schema.
|
|
337
|
+
*/
|
|
338
|
+
async function verifySqlMigration(projectDir, baselineApsorc, migrationFile, cliBin, { seedFile, checkFile, acceptLoss = [] }) {
|
|
339
|
+
const { PGlite } = await Promise.resolve().then(() => tslib_1.__importStar(require("@electric-sql/pglite")));
|
|
340
|
+
const steps = [];
|
|
341
|
+
const done = (s) => {
|
|
342
|
+
steps.push(s);
|
|
343
|
+
return s.ok;
|
|
344
|
+
};
|
|
345
|
+
let sql;
|
|
346
|
+
try {
|
|
347
|
+
sql = (0, migration_file_1.migrationSql)(migrationFile, fs.readFileSync(path.resolve(projectDir, migrationFile), "utf8"));
|
|
348
|
+
}
|
|
349
|
+
catch (error) {
|
|
350
|
+
done({ name: "read", ok: false, output: error.message });
|
|
351
|
+
return { ok: false, steps };
|
|
352
|
+
}
|
|
353
|
+
const findings = (0, review_1.reviewMigration)((0, migration_file_1.typeormMigration)("Migration", [sql.up], { down: [] }));
|
|
354
|
+
const dataDir = fs.mkdtempSync(path.join(os.tmpdir(), "apso-pglite-"));
|
|
355
|
+
// The CLI's own PGlite: the project has no node_modules to load one from.
|
|
356
|
+
const cliRoot = path.dirname(path.dirname(cliBin));
|
|
357
|
+
const script = (name, code, arg) => run(process.execPath, ["-e", code, ...(arg ? [path.resolve(arg)] : [])], cliRoot, (0, exports.pgliteEnv)(dataDir)).then((r) => {
|
|
358
|
+
done(step(name, r));
|
|
359
|
+
return r;
|
|
360
|
+
});
|
|
361
|
+
const exec = async (name, statements) => {
|
|
362
|
+
const db = new PGlite(dataDir);
|
|
363
|
+
try {
|
|
364
|
+
await db.exec(statements);
|
|
365
|
+
return done({ name, ok: true });
|
|
366
|
+
}
|
|
367
|
+
catch (error) {
|
|
368
|
+
return done({ name, ok: false, output: error.message });
|
|
369
|
+
}
|
|
370
|
+
finally {
|
|
371
|
+
await db.close();
|
|
372
|
+
}
|
|
373
|
+
};
|
|
374
|
+
const drift = async (name, apsorcFile) => {
|
|
375
|
+
const db = new PGlite(dataDir);
|
|
376
|
+
const fresh = new PGlite();
|
|
377
|
+
try {
|
|
378
|
+
await fresh.exec(ddlOf(apsorcFile));
|
|
379
|
+
const diff = (0, catalog_1.catalogDiff)(await (0, catalog_1.readCatalog)(fresh), await (0, catalog_1.readCatalog)(db));
|
|
380
|
+
return done({
|
|
381
|
+
name,
|
|
382
|
+
ok: diff.length === 0,
|
|
383
|
+
output: diff.length > 0 ? `The database differs from ${path.basename(apsorcFile)} (- missing, + extra):\n${diff.join("\n")}` : undefined,
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
finally {
|
|
387
|
+
await db.close();
|
|
388
|
+
await fresh.close();
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
try {
|
|
392
|
+
if (!(await exec("baseline", ddlOf(baselineApsorc))))
|
|
393
|
+
return { ok: false, steps, findings };
|
|
394
|
+
const count = async (name, file) => {
|
|
395
|
+
const r = await script(name, COUNT_ROWS, file);
|
|
396
|
+
return r.ok ? JSON.parse(r.output.trim().split("\n").pop()) : undefined;
|
|
397
|
+
};
|
|
398
|
+
const before = seedFile ? await count("seed", seedFile) : undefined;
|
|
399
|
+
if (seedFile && !before)
|
|
400
|
+
return { ok: false, steps, findings };
|
|
401
|
+
if (!(await exec("migrate", sql.up)))
|
|
402
|
+
return { ok: false, steps, findings };
|
|
403
|
+
if (before) {
|
|
404
|
+
const after = await count("count");
|
|
405
|
+
if (!after)
|
|
406
|
+
return { ok: false, steps, findings };
|
|
407
|
+
const lost = dataLoss(before, after, sql.up);
|
|
408
|
+
const ok = (0, exports.unaccepted)(lost, acceptLoss).length === 0;
|
|
409
|
+
done({
|
|
410
|
+
name: "data",
|
|
411
|
+
ok,
|
|
412
|
+
output: lost.length > 0
|
|
413
|
+
? `The migration lost seeded data:\n${lost.join("\n")}${ok ? "" : "\nFix the migration. If data moved somewhere the check can't follow (e.g. a field split in two), prove it with --check and rerun with --accept-loss <table.column>."}`
|
|
414
|
+
: undefined,
|
|
415
|
+
});
|
|
416
|
+
if (!ok)
|
|
417
|
+
return { ok: false, steps, findings };
|
|
418
|
+
}
|
|
419
|
+
const checked = checkFile ? await script("check", RUN_CHECKS, checkFile) : undefined;
|
|
420
|
+
if (checked && !checked.ok)
|
|
421
|
+
return { ok: false, steps, findings };
|
|
422
|
+
if (!(await drift("drift", path.join(projectDir, ".apsorc"))))
|
|
423
|
+
return { ok: false, steps, findings };
|
|
424
|
+
if (sql.down === undefined)
|
|
425
|
+
done({ name: "down", ok: true, output: "down refuses to run (irreversible), so it was not tested." });
|
|
426
|
+
else if (!(await exec("down", sql.down)) || !(await drift("down drift", baselineApsorc)))
|
|
427
|
+
return { ok: false, steps, findings };
|
|
428
|
+
return { ok: true, steps, findings };
|
|
429
|
+
}
|
|
430
|
+
finally {
|
|
431
|
+
fs.rmSync(dataDir, { recursive: true, force: true });
|
|
432
|
+
}
|
|
433
|
+
}
|
|
209
434
|
const freePort = () => new Promise((resolve, reject) => {
|
|
210
435
|
const s = net.createServer();
|
|
211
436
|
s.listen(0, () => {
|
|
@@ -492,6 +717,7 @@ const cliBinPath = (root) => path.join(root, "bin", "run");
|
|
|
492
717
|
exports.cliBinPath = cliBinPath;
|
|
493
718
|
/** Print a check's steps (or JSON) through the command's logger. */
|
|
494
719
|
function report(log, result, json, summary) {
|
|
720
|
+
var _a;
|
|
495
721
|
if (json) {
|
|
496
722
|
log(JSON.stringify(result));
|
|
497
723
|
return;
|
|
@@ -501,6 +727,8 @@ function report(log, result, json, summary) {
|
|
|
501
727
|
if (s.output)
|
|
502
728
|
log(s.output.replace(/^/gm, " "));
|
|
503
729
|
}
|
|
730
|
+
for (const f of (_a = result.findings) !== null && _a !== void 0 ? _a : [])
|
|
731
|
+
log(`WARN ${f.code}: ${f.message}`);
|
|
504
732
|
if (summary)
|
|
505
733
|
log(summary);
|
|
506
734
|
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Review a TypeORM-generated migration for statements that lose data or fail
|
|
3
|
+
* on a populated database. TypeORM changes a column type by DROP + ADD, casts
|
|
4
|
+
* enums without remapping removed values, and adds NOT NULL columns as if the
|
|
5
|
+
* table were empty. The generated file is kept; each finding names a
|
|
6
|
+
* statement to rewrite.
|
|
7
|
+
*/
|
|
8
|
+
export interface Finding {
|
|
9
|
+
code: string;
|
|
10
|
+
table: string;
|
|
11
|
+
column?: string;
|
|
12
|
+
message: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function reviewMigration(source: string): Finding[];
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Review a TypeORM-generated migration for statements that lose data or fail
|
|
4
|
+
* on a populated database. TypeORM changes a column type by DROP + ADD, casts
|
|
5
|
+
* enums without remapping removed values, and adds NOT NULL columns as if the
|
|
6
|
+
* table were empty. The generated file is kept; each finding names a
|
|
7
|
+
* statement to rewrite.
|
|
8
|
+
*/
|
|
9
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
+
exports.reviewMigration = void 0;
|
|
11
|
+
/** The SQL of each queryRunner.query call in a method body. */
|
|
12
|
+
const queries = (body) => [...body.matchAll(/queryRunner\.query\(\s*`([\S\s]*?)`/g)].map((m) => m[1]);
|
|
13
|
+
const enumValues = (sql) => {
|
|
14
|
+
var _a, _b;
|
|
15
|
+
return new Set([...((_b = (_a = sql === null || sql === void 0 ? void 0 : sql.match(/ENUM\(([^)]*)\)/)) === null || _a === void 0 ? void 0 : _a[1]) !== null && _b !== void 0 ? _b : "").matchAll(/'([^']*)'/g)].map((v) => v[1]));
|
|
16
|
+
};
|
|
17
|
+
const typeSql = (sqls, type) => sqls.find((s) => s.startsWith(`CREATE TYPE "public"."${type}" AS ENUM`));
|
|
18
|
+
const list = (values) => values.map((v) => `"${v}"`).join(", ");
|
|
19
|
+
function reviewMigration(source) {
|
|
20
|
+
var _a, _b, _c, _d, _e;
|
|
21
|
+
const [upBody, downBody = ""] = source.split(/public\s+async\s+down\s*\(/);
|
|
22
|
+
const up = queries(upBody);
|
|
23
|
+
const down = queries(downBody);
|
|
24
|
+
const findings = [];
|
|
25
|
+
const add = (code, table, column, message) => findings.push({ code, table, column, message });
|
|
26
|
+
// Statements on tables this migration creates touch no existing rows.
|
|
27
|
+
const created = new Set(up.map((q) => { var _a; return (_a = q.match(/^CREATE TABLE "([^"]+)"/)) === null || _a === void 0 ? void 0 : _a[1]; }).filter(Boolean));
|
|
28
|
+
const existing = (table) => !created.has(table);
|
|
29
|
+
// An UPDATE or INSERT before a statement that names the column means the
|
|
30
|
+
// migration already moves or backfills its data.
|
|
31
|
+
const handled = (i, col) => up.slice(0, i).some((q) => /^(UPDATE|INSERT)\b/.test(q) && q.includes(`"${col}"`));
|
|
32
|
+
const dropped = new Map();
|
|
33
|
+
const added = new Map();
|
|
34
|
+
for (const [i, q] of up.entries()) {
|
|
35
|
+
const d = q.match(/^ALTER TABLE "([^"]+)" DROP COLUMN "([^"]+)"/);
|
|
36
|
+
if (d && !handled(i, d[2]))
|
|
37
|
+
dropped.set(d[1], [...((_a = dropped.get(d[1])) !== null && _a !== void 0 ? _a : []), d[2]]);
|
|
38
|
+
const a = q.match(/^ALTER TABLE "([^"]+)" ADD "([^"]+)"/);
|
|
39
|
+
if (a)
|
|
40
|
+
added.set(a[1], [...((_b = added.get(a[1])) !== null && _b !== void 0 ? _b : []), a[2]]);
|
|
41
|
+
}
|
|
42
|
+
for (const [table, cols] of dropped) {
|
|
43
|
+
const adds = (_c = added.get(table)) !== null && _c !== void 0 ? _c : [];
|
|
44
|
+
const newCols = adds.filter((a) => !cols.includes(a));
|
|
45
|
+
for (const col of cols) {
|
|
46
|
+
if (adds.includes(col))
|
|
47
|
+
add("TYPE_CHANGE_DROPS_DATA", table, col, `${table}.${col}: TypeORM drops and re-adds the column to change its type, deleting every value. Replace the DROP and ADD with ALTER COLUMN "${col}" TYPE ... USING a cast that converts the existing values.`);
|
|
48
|
+
else if (newCols.length > 0)
|
|
49
|
+
add("POSSIBLE_RENAME", table, col, `${table}.${col} is dropped while ${list(newCols)} is added. If that is a rename, replace the DROP and ADD with RENAME COLUMN to keep the data; otherwise every value in "${col}" is deleted.`);
|
|
50
|
+
else
|
|
51
|
+
add("DROPS_COLUMN", table, col, created.size > 0
|
|
52
|
+
? `${table}.${col} is dropped with all its values while this migration creates ${list([...created])}. If the data moves there, copy it with INSERT ... SELECT before the DROP; otherwise confirm the loss.`
|
|
53
|
+
: `${table}.${col} is dropped with all its values. Confirm the loss, or copy the data first.`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
for (const [i, q] of up.entries()) {
|
|
57
|
+
let m = q.match(/^DROP TABLE "([^"]+)"/);
|
|
58
|
+
if (m)
|
|
59
|
+
add("DROPS_TABLE", m[1], undefined, `Table ${m[1]} is dropped with all its rows. Confirm the loss, or rename the table if an entity was renamed.`);
|
|
60
|
+
m = q.match(/^ALTER TABLE "([^"]+)" ADD "([^"]+)" [^,]*\bNOT NULL\b(.*)$/);
|
|
61
|
+
if (m && existing(m[1])) {
|
|
62
|
+
const def = (_d = m[3].match(/DEFAULT (.+)$/)) === null || _d === void 0 ? void 0 : _d[1].trim();
|
|
63
|
+
if (def)
|
|
64
|
+
add("DEFAULT_BACKFILLS_ROWS", m[1], m[2], `${m[1]}.${m[2]} is added NOT NULL DEFAULT ${def}, so every existing row gets ${def}. Confirm that is right for existing rows, or add the column nullable, backfill it, then SET NOT NULL.`);
|
|
65
|
+
else
|
|
66
|
+
add("NOT_NULL_WITHOUT_BACKFILL", m[1], m[2], `${m[1]}.${m[2]} is added NOT NULL with no default, which fails if ${m[1]} has any rows. Add it nullable, backfill it, then SET NOT NULL.`);
|
|
67
|
+
}
|
|
68
|
+
m = q.match(/^ALTER TABLE "([^"]+)" ALTER COLUMN "([^"]+)" SET NOT NULL/);
|
|
69
|
+
if (m && existing(m[1]) && !handled(i, m[2]))
|
|
70
|
+
add("SET_NOT_NULL", m[1], m[2], `${m[1]}.${m[2]} becomes NOT NULL, which fails on any row where it is null. Backfill those rows first.`);
|
|
71
|
+
m = q.match(/^ALTER TABLE "([^"]+)" ALTER COLUMN "([^"]+)" TYPE (?!"public"\."[^"]+_enum")/);
|
|
72
|
+
if (m && existing(m[1]))
|
|
73
|
+
add("TYPE_CAST", m[1], m[2], `${m[1]}.${m[2]} changes type in place; a value the cast cannot convert fails the migration. Check the USING clause handles every existing value.`);
|
|
74
|
+
m =
|
|
75
|
+
(_e = q.match(/^CREATE UNIQUE INDEX "[^"]+" ON "([^"]+)" \(([^)]+)\)/)) !== null && _e !== void 0 ? _e : q.match(/^ALTER TABLE "([^"]+)" ADD CONSTRAINT "[^"]+" UNIQUE \(([^)]+)\)/);
|
|
76
|
+
if (m && existing(m[1]))
|
|
77
|
+
add("UNIQUE_ON_EXISTING_ROWS", m[1], m[2].replace(/"/g, ""), `A unique constraint on ${m[1]} (${m[2].replace(/"/g, "")}) fails if existing rows hold duplicates. Check for duplicates first and decide which row wins.`);
|
|
78
|
+
m = q.match(/^ALTER TABLE "([^"]+)" ADD CONSTRAINT "[^"]+" FOREIGN KEY \("([^"]+)"\) REFERENCES "([^"]+)"/);
|
|
79
|
+
if (m && existing(m[1]) && created.has(m[3]))
|
|
80
|
+
add("FOREIGN_KEY_TO_NEW_TABLE", m[1], m[2], `Existing ${m[1]} rows must point at ${m[3]}, which this migration creates empty, so the foreign key fails on any row with ${m[2]} set. If ${m[3]} replaces an existing entity, replace the CREATE TABLE with ALTER TABLE "<old table>" RENAME TO "${m[3]}" and ALTER SEQUENCE "<old table>_id_seq" RENAME TO "${m[3]}_id_seq"; TypeORM leaves the old table in place.`);
|
|
81
|
+
}
|
|
82
|
+
// TypeORM swaps an enum by renaming the old type to *_old, creating the new
|
|
83
|
+
// one and casting through text; down() recreates *_old with the old values.
|
|
84
|
+
for (const q of up) {
|
|
85
|
+
const m = q.match(/^ALTER TYPE "public"\."([^"]+)" RENAME TO "([^"]+_old)"/);
|
|
86
|
+
if (!m)
|
|
87
|
+
continue;
|
|
88
|
+
const now = enumValues(typeSql(up, m[1]));
|
|
89
|
+
const cast = up.find((s) => s.includes(`TYPE "public"."${m[1]}" USING`));
|
|
90
|
+
// A USING clause that names a removed value maps it; the rest still fail.
|
|
91
|
+
const removed = [...enumValues(typeSql(down, m[2]))].filter((v) => !now.has(v) && !(cast === null || cast === void 0 ? void 0 : cast.includes(`'${v}'`)));
|
|
92
|
+
const col = cast === null || cast === void 0 ? void 0 : cast.match(/^ALTER TABLE "([^"]+)" ALTER COLUMN "([^"]+)"/);
|
|
93
|
+
if (removed.length > 0 && col)
|
|
94
|
+
add("ENUM_VALUES_REMOVED", col[1], col[2], `${col[1]}.${col[2]} drops enum value ${removed.map((v) => `'${v}'`).join(", ")}; the cast fails on any row that still holds it. Map those rows to a remaining value in the USING clause, e.g. CASE "${col[2]}"::text WHEN '${removed[0]}' THEN '...' ELSE "${col[2]}"::text END.`);
|
|
95
|
+
}
|
|
96
|
+
return findings;
|
|
97
|
+
}
|
|
98
|
+
exports.reviewMigration = reviewMigration;
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@apso/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.44.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@apso/cli",
|
|
9
|
-
"version": "0.
|
|
9
|
+
"version": "0.44.0",
|
|
10
10
|
"license": "Apache-2.0",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@apso/schema-tools": "^0.2.0",
|