@stacksjs/database 0.70.251 → 0.70.253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1 -0
- package/dist/migration-ledger.d.ts +189 -0
- package/dist/migration-ledger.js +382 -0
- package/package.json +10 -10
package/dist/index.d.ts
CHANGED
|
@@ -107,6 +107,10 @@ export * from './defaults';
|
|
|
107
107
|
// Dialect classification for the committed migration corpus, so a corpus
|
|
108
108
|
// emitted for one database fails loudly before a single statement runs.
|
|
109
109
|
export * from './migration-dialect';
|
|
110
|
+
// Ledger drift audit (stacksjs/stacks#2203) — compare the corpus on disk, the
|
|
111
|
+
// `migrations` table, and the live schema, because regeneration renumbers files
|
|
112
|
+
// and the ledger keys on the filename.
|
|
113
|
+
export * from './migration-ledger';
|
|
110
114
|
// Model resolution for the generator: userland + framework defaults, flattened
|
|
111
115
|
// because bun-query-builder's loadModels reads only the top level of a dir.
|
|
112
116
|
export * from './model-sources';
|
package/dist/index.js
CHANGED
|
@@ -30,6 +30,7 @@ export { migrateRbacTables } from "./rbac-tables";
|
|
|
30
30
|
export * from "./sql-helpers";
|
|
31
31
|
export * from "./defaults";
|
|
32
32
|
export * from "./migration-dialect";
|
|
33
|
+
export * from "./migration-ledger";
|
|
33
34
|
export * from "./model-sources";
|
|
34
35
|
export * from "./ensure-database";
|
|
35
36
|
export { auditForeignKeys, findFkOrphans, getDeclaredFKs, getLiveFKs } from "./fk-audit";
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import process from 'node:process';
|
|
2
|
+
/**
|
|
3
|
+
* Strip a migration's SQL down to something safe to pattern-match.
|
|
4
|
+
*
|
|
5
|
+
* Blanks comments and single-quoted string literals while PRESERVING
|
|
6
|
+
* double-quoted identifiers, which is the opposite of what
|
|
7
|
+
* {@link stripSqlNoise} in `migration-dialect.ts` wants — that one is matching
|
|
8
|
+
* dialect markers and has no use for names, whereas every effect here IS a
|
|
9
|
+
* name. Blanking the literals still matters: without it a data migration whose
|
|
10
|
+
* payload happens to contain `CREATE TABLE "x"` would register as creating a
|
|
11
|
+
* table, and then be silently recorded as applied.
|
|
12
|
+
*
|
|
13
|
+
* Blanking preserves offsets and line count, so nothing downstream has to care.
|
|
14
|
+
*/
|
|
15
|
+
export declare function stripForEffects(sql: string): string;
|
|
16
|
+
/**
|
|
17
|
+
* The migration's identity, independent of where it sits in the sequence.
|
|
18
|
+
*
|
|
19
|
+
* This is the whole basis for reconciliation: `0000000003-create-issues-table`
|
|
20
|
+
* and `0000000002-create-issues-table` are the same migration, and the ledger
|
|
21
|
+
* only failed to see that because it stored the ordinal.
|
|
22
|
+
*/
|
|
23
|
+
export declare function logicalName(file: string): string;
|
|
24
|
+
/** Schema changes a migration file makes that the live database can confirm. */
|
|
25
|
+
export declare function migrationEffects(sql: string): MigrationEffect[];
|
|
26
|
+
/**
|
|
27
|
+
* Effects this dialect can actually confirm.
|
|
28
|
+
*
|
|
29
|
+
* SQLite has no named constraints reachable by introspection (foreign keys are
|
|
30
|
+
* inline on CREATE TABLE) and no user-defined types, and the runner
|
|
31
|
+
* deliberately records ADD CONSTRAINT / CREATE TYPE files as executed WITHOUT
|
|
32
|
+
* running them so a later `DB_CONNECTION` flip can replay the file
|
|
33
|
+
* (stacksjs/stacks#1916). Checking for those effects on SQLite would therefore
|
|
34
|
+
* report every such file as `reverted`, which is both wrong and loud. MySQL has
|
|
35
|
+
* no standalone enum type either.
|
|
36
|
+
*/
|
|
37
|
+
export declare function verifiableEffects(effects: MigrationEffect[], dialect: LedgerDialect): MigrationEffect[];
|
|
38
|
+
/** Read the parts of the live schema an effect can be checked against. */
|
|
39
|
+
export declare function readLiveSchema(dialect: LedgerDialect, runner?: SqlRunner): Promise<LiveSchema>;
|
|
40
|
+
/** Whether a single effect can be found in the live schema. */
|
|
41
|
+
export declare function effectPresent(effect: MigrationEffect, schema: LiveSchema): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Decide what a single migration file's state actually is.
|
|
44
|
+
*
|
|
45
|
+
* Pure, so the interesting cases are testable without a database. The order
|
|
46
|
+
* matters: "no verifiable effects" has to be answered before "all present",
|
|
47
|
+
* because vacuously-all-present is exactly the wrong answer for a data
|
|
48
|
+
* migration — it is how a `DELETE FROM oauth_access_tokens` gets recorded as
|
|
49
|
+
* applied on the strength of having nothing to check.
|
|
50
|
+
*/
|
|
51
|
+
export declare function classifyMigration(recorded: boolean, present: MigrationEffect[], absent: MigrationEffect[]): MigrationStatus;
|
|
52
|
+
/**
|
|
53
|
+
* Every filename the `migrations` table has recorded.
|
|
54
|
+
*
|
|
55
|
+
* An absent table is a legitimate state (nothing has ever migrated), so it
|
|
56
|
+
* reads as an empty ledger rather than an error.
|
|
57
|
+
*/
|
|
58
|
+
export declare function readLedger(runner?: SqlRunner): Promise<string[]>;
|
|
59
|
+
/**
|
|
60
|
+
* Match ledger rows to disk files by logical name, so a renumbered corpus can
|
|
61
|
+
* have its bookkeeping rewritten instead of being re-run.
|
|
62
|
+
*
|
|
63
|
+
* Pure — takes the two lists and returns a plan. Refuses anything it cannot
|
|
64
|
+
* prove: a logical name appearing on more than one disk file, or two ledger
|
|
65
|
+
* rows converging on one file, are both reported as ambiguous rather than
|
|
66
|
+
* guessed at. A wrong remap silently un-applies a migration, which is the very
|
|
67
|
+
* failure this exists to fix.
|
|
68
|
+
*/
|
|
69
|
+
export declare function planLedgerRemap(ledger: string[], diskFiles: string[]): LedgerRemapPlan;
|
|
70
|
+
/**
|
|
71
|
+
* Compare the migration corpus, the ledger, and the live schema.
|
|
72
|
+
*
|
|
73
|
+
* Read-only. Nothing here writes, so it is safe to run on production and safe
|
|
74
|
+
* to wire into `buddy doctor`.
|
|
75
|
+
*/
|
|
76
|
+
export declare function auditMigrationLedger(options?: {
|
|
77
|
+
dir?: string
|
|
78
|
+
/** Audit a specific dialect instead of the configured one. */
|
|
79
|
+
dialect?: LedgerDialect
|
|
80
|
+
/** Audit a database other than the process-wide one. */
|
|
81
|
+
run?: SqlRunner
|
|
82
|
+
}): Promise<MigrationLedgerAudit>;
|
|
83
|
+
/**
|
|
84
|
+
* Bring the ledger back in line with what the schema proves.
|
|
85
|
+
*
|
|
86
|
+
* Two operations, both conservative:
|
|
87
|
+
*
|
|
88
|
+
* 1. **Remap** a ledger row onto its renumbered file. Nothing runs; only the
|
|
89
|
+
* recorded name changes. This is the direct undo of #2203.
|
|
90
|
+
* 2. **Record** a `stranded` file — one whose every effect is already in the
|
|
91
|
+
* schema — so the runner stops treating it as pending.
|
|
92
|
+
*
|
|
93
|
+
* Everything else is refused and reported. `partial` files have half-applied
|
|
94
|
+
* effects and no safe automatic answer; `unverifiable` ones (pure DML, like the
|
|
95
|
+
* `DELETE FROM oauth_access_tokens` token revocation in the shipped corpus)
|
|
96
|
+
* leave no trace to check, and recording one on a hunch would skip a migration
|
|
97
|
+
* that never ran. Those are exactly the cases worth a human's attention, which
|
|
98
|
+
* is why they are listed rather than silently handled.
|
|
99
|
+
*/
|
|
100
|
+
export declare function reconcileMigrationLedger(options?: {
|
|
101
|
+
dir?: string
|
|
102
|
+
/** Report what would change without writing. */
|
|
103
|
+
dryRun?: boolean
|
|
104
|
+
/** Also record `partial` files. Off by default, and rarely right. */
|
|
105
|
+
includePartial?: boolean
|
|
106
|
+
/** Reconcile a specific dialect instead of the configured one. */
|
|
107
|
+
dialect?: LedgerDialect
|
|
108
|
+
/** Reconcile a database other than the process-wide one. */
|
|
109
|
+
run?: SqlRunner
|
|
110
|
+
}): Promise<ReconcileResult>;
|
|
111
|
+
/**
|
|
112
|
+
* A schema change a migration makes that can be confirmed by looking at the
|
|
113
|
+
* live database. Deliberately narrow: only effects whose presence is
|
|
114
|
+
* unambiguous. An `ALTER COLUMN ... TYPE`, an `UPDATE`, or a `DELETE` leaves no
|
|
115
|
+
* such trace, and guessing at those is how you end up recording a data
|
|
116
|
+
* migration that never ran.
|
|
117
|
+
*/
|
|
118
|
+
export declare interface MigrationEffect {
|
|
119
|
+
kind: 'table' | 'column' | 'index' | 'constraint' | 'enum'
|
|
120
|
+
table?: string
|
|
121
|
+
name: string
|
|
122
|
+
}
|
|
123
|
+
export declare interface MigrationLedgerEntry {
|
|
124
|
+
file: string
|
|
125
|
+
logical: string
|
|
126
|
+
recorded: boolean
|
|
127
|
+
status: MigrationStatus
|
|
128
|
+
effects: MigrationEffect[]
|
|
129
|
+
present: MigrationEffect[]
|
|
130
|
+
absent: MigrationEffect[]
|
|
131
|
+
}
|
|
132
|
+
export declare interface LedgerOrphan {
|
|
133
|
+
migration: string
|
|
134
|
+
renamedTo?: string
|
|
135
|
+
}
|
|
136
|
+
export declare interface MigrationLedgerAudit {
|
|
137
|
+
supported: boolean
|
|
138
|
+
dialect: LedgerDialect | 'other'
|
|
139
|
+
dir: string
|
|
140
|
+
entries: MigrationLedgerEntry[]
|
|
141
|
+
orphans: LedgerOrphan[]
|
|
142
|
+
counts: Record<MigrationStatus, number>
|
|
143
|
+
recordedCount: number
|
|
144
|
+
remapPlan: LedgerRemapPlan
|
|
145
|
+
drift: boolean
|
|
146
|
+
}
|
|
147
|
+
export declare interface LedgerRemap {
|
|
148
|
+
from: string
|
|
149
|
+
to: string
|
|
150
|
+
}
|
|
151
|
+
export declare interface LedgerRemapPlan {
|
|
152
|
+
remap: LedgerRemap[]
|
|
153
|
+
ambiguous: string[]
|
|
154
|
+
dropped: string[]
|
|
155
|
+
}
|
|
156
|
+
export declare interface LiveSchema {
|
|
157
|
+
tables: Set<string>
|
|
158
|
+
columns: Map<string, Set<string>>
|
|
159
|
+
indexes: Set<string>
|
|
160
|
+
constraints: Set<string>
|
|
161
|
+
enums: Set<string>
|
|
162
|
+
}
|
|
163
|
+
export declare interface ReconcileResult {
|
|
164
|
+
remapped: LedgerRemap[]
|
|
165
|
+
recorded: string[]
|
|
166
|
+
skipped: Array<{ file: string, reason: string }>
|
|
167
|
+
}
|
|
168
|
+
export type LedgerDialect = 'sqlite' | 'mysql' | 'postgres';
|
|
169
|
+
export type MigrationStatus = | 'applied'
|
|
170
|
+
/** Not recorded, but every effect is already present — a renumber victim. */
|
|
171
|
+
| 'stranded'
|
|
172
|
+
/** Not recorded, and no effect is present — genuinely queued to run. */
|
|
173
|
+
| 'pending'
|
|
174
|
+
/** Not recorded, and only some effects are present — needs a human. */
|
|
175
|
+
| 'partial'
|
|
176
|
+
/** Nothing schema-visible to check (pure DML). Status cannot be inferred. */
|
|
177
|
+
| 'unverifiable'
|
|
178
|
+
/** Recorded, but effects are missing — the schema drifted away from history. */
|
|
179
|
+
| 'reverted';
|
|
180
|
+
/**
|
|
181
|
+
* Runs one SQL string and returns its rows.
|
|
182
|
+
*
|
|
183
|
+
* Injectable for the same reason `ensure-database.ts` takes its own `connect`:
|
|
184
|
+
* everything here has to be exercisable against a database the caller controls.
|
|
185
|
+
* The default binds to the process-wide `db`, which is a single shared handle —
|
|
186
|
+
* fine in production, useless for a test that needs to build a specific
|
|
187
|
+
* drift state, and unable to audit a database other than the configured one.
|
|
188
|
+
*/
|
|
189
|
+
export type SqlRunner = (sql: string) => Promise<any[]>;
|
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import { existsSync, readdirSync, readFileSync } from "node:fs";
|
|
2
|
+
import process from "node:process";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
const SAFE_MIGRATION_FILE = /^[\w.-]+\.sql$/, IDENT = String.raw`["\`\[]?([A-Za-z_]\w*)["\`\]]?`;
|
|
5
|
+
export function stripForEffects(sql) {
|
|
6
|
+
let out = "", i = 0;
|
|
7
|
+
const blank = (text) => text.replace(/[^\n]/g, " ");
|
|
8
|
+
while (i < sql.length) {
|
|
9
|
+
const rest = sql.slice(i), line = rest.match(/^--[^\n]*/);
|
|
10
|
+
if (line) {
|
|
11
|
+
out += blank(line[0]);
|
|
12
|
+
i += line[0].length;
|
|
13
|
+
continue;
|
|
14
|
+
}
|
|
15
|
+
if (rest.startsWith("/*")) {
|
|
16
|
+
const end = rest.indexOf("*/"), chunk = end === -1 ? rest : rest.slice(0, end + 2);
|
|
17
|
+
out += blank(chunk);
|
|
18
|
+
i += chunk.length;
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
if (rest[0] === "'") {
|
|
22
|
+
let j = 1;
|
|
23
|
+
while (j < rest.length && rest[j] !== "'")
|
|
24
|
+
j++;
|
|
25
|
+
const chunk = rest.slice(0, Math.min(j + 1, rest.length));
|
|
26
|
+
out += blank(chunk);
|
|
27
|
+
i += chunk.length;
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
out += sql[i];
|
|
31
|
+
i += 1;
|
|
32
|
+
}
|
|
33
|
+
return out;
|
|
34
|
+
}
|
|
35
|
+
function statementsOf(sql) {
|
|
36
|
+
return stripForEffects(sql).split(";").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
37
|
+
}
|
|
38
|
+
export function logicalName(file) {
|
|
39
|
+
return file.replace(/^\d+[-_]/, "").replace(/\.sql$/i, "");
|
|
40
|
+
}
|
|
41
|
+
export function migrationEffects(sql) {
|
|
42
|
+
const effects = [], seen = new Set, renamedAway = new Set, push = (effect) => {
|
|
43
|
+
const key = effectKey(effect);
|
|
44
|
+
if (seen.has(key))
|
|
45
|
+
return;
|
|
46
|
+
seen.add(key);
|
|
47
|
+
effects.push(effect);
|
|
48
|
+
};
|
|
49
|
+
for (const statement of statementsOf(sql)) {
|
|
50
|
+
const create = new RegExp(String.raw`^CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?${IDENT}`, "i").exec(statement);
|
|
51
|
+
if (create?.[1]) {
|
|
52
|
+
push({ kind: "table", name: create[1] });
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
const rename = new RegExp(String.raw`^ALTER\s+TABLE\s+${IDENT}\s+RENAME\s+TO\s+${IDENT}`, "i").exec(statement);
|
|
56
|
+
if (rename?.[2]) {
|
|
57
|
+
if (rename[1])
|
|
58
|
+
renamedAway.add(rename[1].toLowerCase());
|
|
59
|
+
push({ kind: "table", name: rename[2] });
|
|
60
|
+
continue;
|
|
61
|
+
}
|
|
62
|
+
const index = new RegExp(String.raw`^CREATE\s+(?:UNIQUE\s+)?INDEX\s+(?:IF\s+NOT\s+EXISTS\s+)?${IDENT}`, "i").exec(statement);
|
|
63
|
+
if (index?.[1]) {
|
|
64
|
+
push({ kind: "index", name: index[1] });
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const enumType = new RegExp(String.raw`^CREATE\s+TYPE\s+${IDENT}\s+AS\s+ENUM`, "i").exec(statement);
|
|
68
|
+
if (enumType?.[1]) {
|
|
69
|
+
push({ kind: "enum", name: enumType[1] });
|
|
70
|
+
continue;
|
|
71
|
+
}
|
|
72
|
+
const alter = new RegExp(String.raw`^ALTER\s+TABLE\s+(?:IF\s+EXISTS\s+)?${IDENT}\s+(.*)$`, "is").exec(statement);
|
|
73
|
+
if (!alter?.[1] || !alter[2])
|
|
74
|
+
continue;
|
|
75
|
+
const table = alter[1], addColumn = new RegExp(String.raw`\bADD\s+COLUMN\s+(?:IF\s+NOT\s+EXISTS\s+)?${IDENT}`, "gi");
|
|
76
|
+
for (const m of alter[2].matchAll(addColumn))
|
|
77
|
+
if (m[1])
|
|
78
|
+
push({ kind: "column", table, name: m[1] });
|
|
79
|
+
const addConstraint = new RegExp(String.raw`\bADD\s+CONSTRAINT\s+(?:IF\s+NOT\s+EXISTS\s+)?${IDENT}`, "gi");
|
|
80
|
+
for (const m of alter[2].matchAll(addConstraint))
|
|
81
|
+
if (m[1])
|
|
82
|
+
push({ kind: "constraint", table, name: m[1] });
|
|
83
|
+
const addBare = new RegExp(String.raw`\bADD\s+(?!COLUMN\b|CONSTRAINT\b|INDEX\b|KEY\b|PRIMARY\b|UNIQUE\b|FOREIGN\b|FULLTEXT\b|SPATIAL\b|CHECK\b)${IDENT}\s+\w`, "gi");
|
|
84
|
+
for (const m of alter[2].matchAll(addBare))
|
|
85
|
+
if (m[1])
|
|
86
|
+
push({ kind: "column", table, name: m[1] });
|
|
87
|
+
}
|
|
88
|
+
if (renamedAway.size === 0)
|
|
89
|
+
return effects;
|
|
90
|
+
return effects.filter((effect) => {
|
|
91
|
+
const owner = (effect.kind === "table" ? effect.name : effect.table ?? "").toLowerCase();
|
|
92
|
+
return !renamedAway.has(owner);
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
function effectKey(effect) {
|
|
96
|
+
return `${effect.kind}:${(effect.table ?? "").toLowerCase()}.${effect.name.toLowerCase()}`;
|
|
97
|
+
}
|
|
98
|
+
export function verifiableEffects(effects, dialect) {
|
|
99
|
+
if (dialect === "postgres")
|
|
100
|
+
return effects;
|
|
101
|
+
if (dialect === "mysql")
|
|
102
|
+
return effects.filter((e) => e.kind !== "enum");
|
|
103
|
+
return effects.filter((e) => e.kind !== "constraint" && e.kind !== "enum");
|
|
104
|
+
}
|
|
105
|
+
function emptySchema() {
|
|
106
|
+
return { tables: new Set, columns: new Map, indexes: new Set, constraints: new Set, enums: new Set };
|
|
107
|
+
}
|
|
108
|
+
function rowsOf(result) {
|
|
109
|
+
return Array.isArray(result) ? result : [];
|
|
110
|
+
}
|
|
111
|
+
async function defaultRunner() {
|
|
112
|
+
const { db } = await import("./utils");
|
|
113
|
+
return async (sql) => rowsOf(await db.unsafe(sql).execute());
|
|
114
|
+
}
|
|
115
|
+
function pick(row, ...keys) {
|
|
116
|
+
for (const key of keys) {
|
|
117
|
+
const value = row?.[key] ?? row?.[key.toLowerCase()] ?? row?.[key.toUpperCase()];
|
|
118
|
+
if (typeof value === "string" && value.length > 0)
|
|
119
|
+
return value;
|
|
120
|
+
}
|
|
121
|
+
return "";
|
|
122
|
+
}
|
|
123
|
+
export async function readLiveSchema(dialect, runner) {
|
|
124
|
+
const schema = emptySchema(), run = runner ?? await defaultRunner();
|
|
125
|
+
if (dialect === "sqlite") {
|
|
126
|
+
for (const row of await run("SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'")) {
|
|
127
|
+
const name = pick(row, "name");
|
|
128
|
+
if (name)
|
|
129
|
+
schema.tables.add(name.toLowerCase());
|
|
130
|
+
}
|
|
131
|
+
for (const row of await run("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'")) {
|
|
132
|
+
const name = pick(row, "name");
|
|
133
|
+
if (name)
|
|
134
|
+
schema.indexes.add(name.toLowerCase());
|
|
135
|
+
}
|
|
136
|
+
for (const table of schema.tables) {
|
|
137
|
+
if (!/^[a-z_]\w*$/i.test(table))
|
|
138
|
+
continue;
|
|
139
|
+
const cols = new Set;
|
|
140
|
+
for (const row of await run(`PRAGMA table_info("${table}")`)) {
|
|
141
|
+
const name = pick(row, "name");
|
|
142
|
+
if (name)
|
|
143
|
+
cols.add(name.toLowerCase());
|
|
144
|
+
}
|
|
145
|
+
schema.columns.set(table, cols);
|
|
146
|
+
}
|
|
147
|
+
return schema;
|
|
148
|
+
}
|
|
149
|
+
if (dialect === "mysql") {
|
|
150
|
+
for (const row of await run("SELECT TABLE_NAME AS name FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE()")) {
|
|
151
|
+
const name = pick(row, "name", "TABLE_NAME");
|
|
152
|
+
if (name)
|
|
153
|
+
schema.tables.add(name.toLowerCase());
|
|
154
|
+
}
|
|
155
|
+
for (const row of await run("SELECT TABLE_NAME, COLUMN_NAME FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE()")) {
|
|
156
|
+
const table = pick(row, "TABLE_NAME").toLowerCase(), column = pick(row, "COLUMN_NAME").toLowerCase();
|
|
157
|
+
if (!table || !column)
|
|
158
|
+
continue;
|
|
159
|
+
if (!schema.columns.has(table))
|
|
160
|
+
schema.columns.set(table, new Set);
|
|
161
|
+
schema.columns.get(table).add(column);
|
|
162
|
+
}
|
|
163
|
+
for (const row of await run("SELECT DISTINCT INDEX_NAME AS name FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE()")) {
|
|
164
|
+
const name = pick(row, "name", "INDEX_NAME");
|
|
165
|
+
if (name)
|
|
166
|
+
schema.indexes.add(name.toLowerCase());
|
|
167
|
+
}
|
|
168
|
+
for (const row of await run("SELECT CONSTRAINT_NAME AS name FROM information_schema.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = DATABASE()")) {
|
|
169
|
+
const name = pick(row, "name", "CONSTRAINT_NAME");
|
|
170
|
+
if (name)
|
|
171
|
+
schema.constraints.add(name.toLowerCase());
|
|
172
|
+
}
|
|
173
|
+
return schema;
|
|
174
|
+
}
|
|
175
|
+
for (const row of await run("SELECT tablename AS name FROM pg_tables WHERE schemaname = 'public'")) {
|
|
176
|
+
const name = pick(row, "name", "tablename");
|
|
177
|
+
if (name)
|
|
178
|
+
schema.tables.add(name.toLowerCase());
|
|
179
|
+
}
|
|
180
|
+
for (const row of await run("SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = 'public'")) {
|
|
181
|
+
const table = pick(row, "table_name").toLowerCase(), column = pick(row, "column_name").toLowerCase();
|
|
182
|
+
if (!table || !column)
|
|
183
|
+
continue;
|
|
184
|
+
if (!schema.columns.has(table))
|
|
185
|
+
schema.columns.set(table, new Set);
|
|
186
|
+
schema.columns.get(table).add(column);
|
|
187
|
+
}
|
|
188
|
+
for (const row of await run("SELECT indexname AS name FROM pg_indexes WHERE schemaname = 'public'")) {
|
|
189
|
+
const name = pick(row, "name", "indexname");
|
|
190
|
+
if (name)
|
|
191
|
+
schema.indexes.add(name.toLowerCase());
|
|
192
|
+
}
|
|
193
|
+
for (const row of await run("SELECT c.conname AS name FROM pg_constraint c JOIN pg_namespace n ON n.oid = c.connamespace WHERE n.nspname = 'public'")) {
|
|
194
|
+
const name = pick(row, "name", "conname");
|
|
195
|
+
if (name)
|
|
196
|
+
schema.constraints.add(name.toLowerCase());
|
|
197
|
+
}
|
|
198
|
+
for (const row of await run("SELECT t.typname AS name FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace WHERE t.typtype = 'e' AND n.nspname = 'public'")) {
|
|
199
|
+
const name = pick(row, "name", "typname");
|
|
200
|
+
if (name)
|
|
201
|
+
schema.enums.add(name.toLowerCase());
|
|
202
|
+
}
|
|
203
|
+
return schema;
|
|
204
|
+
}
|
|
205
|
+
export function effectPresent(effect, schema) {
|
|
206
|
+
const name = effect.name.toLowerCase();
|
|
207
|
+
switch (effect.kind) {
|
|
208
|
+
case "table":
|
|
209
|
+
return schema.tables.has(name);
|
|
210
|
+
case "column":
|
|
211
|
+
return schema.columns.get((effect.table ?? "").toLowerCase())?.has(name) ?? !1;
|
|
212
|
+
case "index":
|
|
213
|
+
return schema.indexes.has(name);
|
|
214
|
+
case "constraint":
|
|
215
|
+
return schema.constraints.has(name);
|
|
216
|
+
case "enum":
|
|
217
|
+
return schema.enums.has(name);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
export function classifyMigration(recorded, present, absent) {
|
|
221
|
+
const verifiable = present.length + absent.length;
|
|
222
|
+
if (recorded) {
|
|
223
|
+
if (verifiable === 0 || absent.length === 0)
|
|
224
|
+
return "applied";
|
|
225
|
+
return "reverted";
|
|
226
|
+
}
|
|
227
|
+
if (verifiable === 0)
|
|
228
|
+
return "unverifiable";
|
|
229
|
+
if (absent.length === 0)
|
|
230
|
+
return "stranded";
|
|
231
|
+
if (present.length === 0)
|
|
232
|
+
return "pending";
|
|
233
|
+
return "partial";
|
|
234
|
+
}
|
|
235
|
+
function migrationsDir(dir) {
|
|
236
|
+
return dir ?? join(process.cwd(), "database", "migrations");
|
|
237
|
+
}
|
|
238
|
+
function listMigrationFiles(dir) {
|
|
239
|
+
if (!existsSync(dir))
|
|
240
|
+
return [];
|
|
241
|
+
try {
|
|
242
|
+
return readdirSync(dir).filter((f) => f.toLowerCase().endsWith(".sql")).sort();
|
|
243
|
+
} catch {
|
|
244
|
+
return [];
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
async function currentDialect() {
|
|
248
|
+
const driver = ((await import("@stacksjs/env")).env?.DB_CONNECTION ?? "sqlite").toLowerCase();
|
|
249
|
+
if (driver === "sqlite" || driver === "mysql" || driver === "postgres")
|
|
250
|
+
return driver;
|
|
251
|
+
return "other";
|
|
252
|
+
}
|
|
253
|
+
export async function readLedger(runner) {
|
|
254
|
+
try {
|
|
255
|
+
return (await (runner ?? await defaultRunner())("SELECT migration FROM migrations")).map((row) => pick(row, "migration")).filter((name) => name.length > 0).sort();
|
|
256
|
+
} catch {
|
|
257
|
+
return [];
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
export function planLedgerRemap(ledger, diskFiles) {
|
|
261
|
+
const onDisk = new Set(diskFiles), byLogical = new Map;
|
|
262
|
+
for (const file of diskFiles) {
|
|
263
|
+
const key = logicalName(file);
|
|
264
|
+
if (!byLogical.has(key))
|
|
265
|
+
byLogical.set(key, []);
|
|
266
|
+
byLogical.get(key).push(file);
|
|
267
|
+
}
|
|
268
|
+
const claimed = new Set(ledger.filter((row) => onDisk.has(row))), remap = [], ambiguous = [], dropped = [], targets = new Map;
|
|
269
|
+
for (const row of ledger) {
|
|
270
|
+
if (onDisk.has(row))
|
|
271
|
+
continue;
|
|
272
|
+
const candidates = (byLogical.get(logicalName(row)) ?? []).filter((f) => !claimed.has(f));
|
|
273
|
+
if (candidates.length === 0) {
|
|
274
|
+
dropped.push(row);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
if (candidates.length > 1) {
|
|
278
|
+
ambiguous.push(row);
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
const to = candidates[0];
|
|
282
|
+
if (!targets.has(to))
|
|
283
|
+
targets.set(to, []);
|
|
284
|
+
targets.get(to).push(row);
|
|
285
|
+
remap.push({ from: row, to });
|
|
286
|
+
}
|
|
287
|
+
const contested = new Set([...targets.entries()].filter(([, rows]) => rows.length > 1).flatMap(([, rows]) => rows));
|
|
288
|
+
if (contested.size === 0)
|
|
289
|
+
return { remap, ambiguous, dropped };
|
|
290
|
+
return {
|
|
291
|
+
remap: remap.filter((r) => !contested.has(r.from)),
|
|
292
|
+
ambiguous: [...ambiguous, ...contested].sort(),
|
|
293
|
+
dropped
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
export async function auditMigrationLedger(options = {}) {
|
|
297
|
+
const dir = migrationsDir(options.dir), dialect = options.dialect ?? await currentDialect(), files = listMigrationFiles(dir), counts = {
|
|
298
|
+
applied: 0,
|
|
299
|
+
stranded: 0,
|
|
300
|
+
pending: 0,
|
|
301
|
+
partial: 0,
|
|
302
|
+
unverifiable: 0,
|
|
303
|
+
reverted: 0
|
|
304
|
+
}, emptyPlan = { remap: [], ambiguous: [], dropped: [] };
|
|
305
|
+
if (dialect === "other")
|
|
306
|
+
return { supported: !1, dialect, dir, entries: [], orphans: [], counts, recordedCount: 0, remapPlan: emptyPlan, drift: !1 };
|
|
307
|
+
const run = options.run ?? await defaultRunner(), ledger = await readLedger(run), recorded = new Set(ledger), schema = await readLiveSchema(dialect, run), entries = [];
|
|
308
|
+
for (const file of files) {
|
|
309
|
+
let sql = "";
|
|
310
|
+
try {
|
|
311
|
+
sql = readFileSync(join(dir, file), "utf8");
|
|
312
|
+
} catch {
|
|
313
|
+
continue;
|
|
314
|
+
}
|
|
315
|
+
const effects = verifiableEffects(migrationEffects(sql), dialect), present = effects.filter((effect) => effectPresent(effect, schema)), absent = effects.filter((effect) => !effectPresent(effect, schema)), isRecorded = recorded.has(file), status = classifyMigration(isRecorded, present, absent);
|
|
316
|
+
counts[status] += 1;
|
|
317
|
+
entries.push({ file, logical: logicalName(file), recorded: isRecorded, status, effects, present, absent });
|
|
318
|
+
}
|
|
319
|
+
const readable = entries.map((entry) => entry.file), remapPlan = planLedgerRemap(ledger, readable), renamedTo = new Map(remapPlan.remap.map((r) => [r.from, r.to])), orphans = ledger.filter((row) => !readable.includes(row)).map((row) => ({ migration: row, renamedTo: renamedTo.get(row) })), drift = counts.stranded > 0 || counts.partial > 0 || counts.reverted > 0 || orphans.length > 0;
|
|
320
|
+
return { supported: !0, dialect, dir, entries, orphans, counts, recordedCount: ledger.length, remapPlan, drift };
|
|
321
|
+
}
|
|
322
|
+
async function ensureLedgerTable(dialect, run) {
|
|
323
|
+
await run(`CREATE TABLE IF NOT EXISTS migrations (${dialect === "postgres" ? "id SERIAL PRIMARY KEY" : dialect === "mysql" ? "id INT AUTO_INCREMENT PRIMARY KEY" : "id INTEGER PRIMARY KEY AUTOINCREMENT"}, migration VARCHAR(255) NOT NULL UNIQUE, executed_at ${dialect === "postgres" ? "TIMESTAMP" : "DATETIME"} DEFAULT CURRENT_TIMESTAMP)`);
|
|
324
|
+
}
|
|
325
|
+
export async function reconcileMigrationLedger(options = {}) {
|
|
326
|
+
const run = options.run ?? await defaultRunner(), audit = await auditMigrationLedger({ dir: options.dir, dialect: options.dialect, run }), result = { remapped: [], recorded: [], skipped: [] };
|
|
327
|
+
if (!audit.supported) {
|
|
328
|
+
result.skipped.push({ file: "*", reason: `dialect "${audit.dialect}" is not audited` });
|
|
329
|
+
return result;
|
|
330
|
+
}
|
|
331
|
+
const plan = audit.remapPlan;
|
|
332
|
+
for (const row of plan.ambiguous)
|
|
333
|
+
result.skipped.push({ file: row, reason: "ledger row matches more than one file by logical name" });
|
|
334
|
+
for (const row of plan.dropped)
|
|
335
|
+
result.skipped.push({ file: row, reason: "recorded migration no longer exists on disk" });
|
|
336
|
+
const toRecord = [];
|
|
337
|
+
for (const entry of audit.entries) {
|
|
338
|
+
if (entry.status === "stranded") {
|
|
339
|
+
toRecord.push(entry.file);
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
if (entry.status === "partial") {
|
|
343
|
+
if (options.includePartial) {
|
|
344
|
+
toRecord.push(entry.file);
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
result.skipped.push({
|
|
348
|
+
file: entry.file,
|
|
349
|
+
reason: `${entry.present.length}/${entry.effects.length} effects present \u2014 resolve by hand, or pass --include-partial`
|
|
350
|
+
});
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
if (entry.status === "reverted")
|
|
354
|
+
result.skipped.push({
|
|
355
|
+
file: entry.file,
|
|
356
|
+
reason: `recorded, but ${entry.absent.length} effect(s) are missing from the schema`
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
const unsafe = (file) => !SAFE_MIGRATION_FILE.test(file);
|
|
360
|
+
for (const { from, to } of plan.remap.filter((r) => unsafe(r.from) || unsafe(r.to)))
|
|
361
|
+
result.skipped.push({ file: unsafe(from) ? from : to, reason: "migration filename is not safe to write to the ledger" });
|
|
362
|
+
for (const file of toRecord.filter(unsafe))
|
|
363
|
+
result.skipped.push({ file, reason: "migration filename is not safe to write to the ledger" });
|
|
364
|
+
const remapped = plan.remap.filter((r) => !unsafe(r.from) && !unsafe(r.to)), recordable = toRecord.filter((file) => !unsafe(file) && !remapped.some((r) => r.to === file));
|
|
365
|
+
if (options.dryRun) {
|
|
366
|
+
result.remapped = remapped;
|
|
367
|
+
result.recorded = recordable;
|
|
368
|
+
return result;
|
|
369
|
+
}
|
|
370
|
+
await ensureLedgerTable(audit.dialect, run);
|
|
371
|
+
for (const { from, to } of remapped) {
|
|
372
|
+
await run(`UPDATE migrations SET migration = '${to}' WHERE migration = '${from}'`);
|
|
373
|
+
result.remapped.push({ from, to });
|
|
374
|
+
}
|
|
375
|
+
for (const file of recordable) {
|
|
376
|
+
if ((await run(`SELECT migration FROM migrations WHERE migration = '${file}'`)).length > 0)
|
|
377
|
+
continue;
|
|
378
|
+
await run(`INSERT INTO migrations (migration) VALUES ('${file}')`);
|
|
379
|
+
result.recorded.push(file);
|
|
380
|
+
}
|
|
381
|
+
return result;
|
|
382
|
+
}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@stacksjs/database",
|
|
3
3
|
"type": "module",
|
|
4
4
|
"sideEffects": false,
|
|
5
|
-
"version": "0.70.
|
|
5
|
+
"version": "0.70.253",
|
|
6
6
|
"description": "The Stacks database integration.",
|
|
7
7
|
"author": "Chris Breuer",
|
|
8
8
|
"contributors": [
|
|
@@ -58,15 +58,15 @@
|
|
|
58
58
|
"dynamodb-tooling": "^0.3.2"
|
|
59
59
|
},
|
|
60
60
|
"devDependencies": {
|
|
61
|
-
"@stacksjs/cli": "0.70.
|
|
62
|
-
"@stacksjs/config": "0.70.
|
|
63
|
-
"@stacksjs/logging": "0.70.
|
|
64
|
-
"@stacksjs/router": "0.70.
|
|
61
|
+
"@stacksjs/cli": "0.70.253",
|
|
62
|
+
"@stacksjs/config": "0.70.253",
|
|
63
|
+
"@stacksjs/logging": "0.70.253",
|
|
64
|
+
"@stacksjs/router": "0.70.253",
|
|
65
65
|
"better-dx": "^0.2.17",
|
|
66
|
-
"@stacksjs/path": "0.70.
|
|
67
|
-
"@stacksjs/query-builder": "0.70.
|
|
68
|
-
"@stacksjs/storage": "0.70.
|
|
69
|
-
"@stacksjs/strings": "0.70.
|
|
70
|
-
"@stacksjs/utils": "0.70.
|
|
66
|
+
"@stacksjs/path": "0.70.253",
|
|
67
|
+
"@stacksjs/query-builder": "0.70.253",
|
|
68
|
+
"@stacksjs/storage": "0.70.253",
|
|
69
|
+
"@stacksjs/strings": "0.70.253",
|
|
70
|
+
"@stacksjs/utils": "0.70.253"
|
|
71
71
|
}
|
|
72
72
|
}
|