@everystack/cli 0.4.83 → 0.4.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/package.json +3 -3
- package/src/cli/apply-execute.ts +61 -1
- package/src/cli/apply-refusals.ts +74 -0
- package/src/cli/commands/db-apply.ts +20 -4
- package/src/cli/commands/db-plan.ts +1 -0
- package/src/cli/derived-apply.ts +17 -0
- package/src/cli/edge-plan.ts +10 -1
- package/src/cli/fk-preflight.ts +331 -0
- package/src/cli/migration-generate.ts +8 -7
- package/src/cli/plain-output.ts +10 -2
- package/src/cli/schema-diff.ts +98 -11
- package/src/cli/schema-fingerprint-v12.ts +239 -0
- package/src/cli/schema-fingerprint.ts +26 -3
- package/src/cli/schema-introspect.ts +23 -0
- package/src/cli/state-apply.ts +160 -13
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FROZEN v12 canonicalizer — the exact live-side hash the CLI produced under
|
|
3
|
+
* FINGERPRINT_VERSION 12, before a NOT VALID foreign key entered the state at v13.
|
|
4
|
+
*
|
|
5
|
+
* DO NOT EDIT this to track new behaviour. It exists for ONE purpose: `db:apply`'s
|
|
6
|
+
* fast-forward proof recomputes a `schema_log` row recorded at `fp_version = 12` against a
|
|
7
|
+
* CURRENT live read, and comparing an old-format hash against a current-format one reports a
|
|
8
|
+
* phantom mismatch. So the descent registry recomputes the LIVE side under the ROW's canon —
|
|
9
|
+
* this function — per the standing rule in `RETAINED_CANONS`.
|
|
10
|
+
*
|
|
11
|
+
* It is a SELF-CONTAINED copy of the v12 canonical form. The ONLY behavioural difference from
|
|
12
|
+
* v13 is the foreign-key slice: v12 hashed a key's columns, target and referential actions and
|
|
13
|
+
* nothing else, so a key pg had never checked against the rows already in the table (`NOT
|
|
14
|
+
* VALID`) hashed IDENTICALLY to the validated key the models declare — a half-applied key
|
|
15
|
+
* reported MATCH. v13 adds the flag. Everything else (the command fan, effective-privilege
|
|
16
|
+
* deadness, the USAGE-only sequence derivation) is byte-identical to v13 and frozen here so a
|
|
17
|
+
* future change to the live form cannot silently move a recorded v12 hash. It reuses only the
|
|
18
|
+
* stable leaf normalizers (deparse + structure) and the authz-contract privilege oracles, whose
|
|
19
|
+
* behaviour is equal across every version.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { createHash } from 'node:crypto';
|
|
23
|
+
import type { PolicyContract, TableContract, SequenceGrantContract } from './authz-contract';
|
|
24
|
+
import { holdsPrivilegeEffective, policyCommands } from './authz-contract';
|
|
25
|
+
import type { SchemaSnapshot, TableSchema } from './schema-introspect';
|
|
26
|
+
import { normalizeDefault, normalizeCheck, indexKey, nextvalSequence } from './schema-diff';
|
|
27
|
+
import { stableStringify, schemaOf, type CanonicalizeOptions } from './schema-fingerprint';
|
|
28
|
+
|
|
29
|
+
const PUBLIC_ROLE = 'public';
|
|
30
|
+
|
|
31
|
+
const byKey = <T>(items: T[], key: (item: T) => string): T[] =>
|
|
32
|
+
[...items].sort((a, b) => (key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0));
|
|
33
|
+
|
|
34
|
+
// --- v12 authz canonical form (frozen: the command fan, identical to v13) ---
|
|
35
|
+
|
|
36
|
+
interface PolicyArm {
|
|
37
|
+
role: string;
|
|
38
|
+
command: string;
|
|
39
|
+
permissive: boolean;
|
|
40
|
+
using: string;
|
|
41
|
+
check: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function policyArms(p: PolicyContract): PolicyArm[] {
|
|
45
|
+
const roles = p.roles.includes(PUBLIC_ROLE) ? [PUBLIC_ROLE] : [...p.roles].sort();
|
|
46
|
+
const arms: PolicyArm[] = [];
|
|
47
|
+
for (const command of policyCommands(p)) {
|
|
48
|
+
const using = command === 'INSERT' ? '' : (p.using ?? '');
|
|
49
|
+
const check = command === 'SELECT' || command === 'DELETE' ? '' : ((p.check ?? p.using) ?? '');
|
|
50
|
+
for (const role of roles) arms.push({ role, command, permissive: p.permissive, using, check });
|
|
51
|
+
}
|
|
52
|
+
return arms;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function armKey(a: PolicyArm): string {
|
|
56
|
+
return `${a.role}|${JSON.stringify([a.command, a.permissive, a.using, a.check])}`;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function multiset(entries: string[]): Array<[string, number]> {
|
|
60
|
+
const counts = new Map<string, number>();
|
|
61
|
+
for (const e of entries) counts.set(e, (counts.get(e) ?? 0) + 1);
|
|
62
|
+
return [...counts.entries()].sort(([a], [b]) => (a < b ? -1 : 1));
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function governedArm(governed: ReadonlySet<string>, role: string): boolean {
|
|
66
|
+
return governed.has(role) || role === PUBLIC_ROLE || (role.toUpperCase() === 'PUBLIC' && governed.has('PUBLIC'));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function armDead(t: TableContract, a: PolicyArm): boolean {
|
|
70
|
+
if (a.role === PUBLIC_ROLE || a.role.toUpperCase() === 'PUBLIC') {
|
|
71
|
+
const grantees = t.effectivePrivileges
|
|
72
|
+
? Object.keys(t.effectivePrivileges)
|
|
73
|
+
: [...new Set([...Object.keys(t.grants), ...Object.keys(t.columnGrants ?? {})])];
|
|
74
|
+
return !grantees.some((r) => holdsPrivilegeEffective(t, r, a.command));
|
|
75
|
+
}
|
|
76
|
+
return !holdsPrivilegeEffective(t, a.role, a.command);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function armSubsumed(a: PolicyArm, alive: readonly PolicyArm[]): boolean {
|
|
80
|
+
if (!a.permissive || a.role === PUBLIC_ROLE || a.role.toUpperCase() === 'PUBLIC') return false;
|
|
81
|
+
return alive.some((o) =>
|
|
82
|
+
(o.role === PUBLIC_ROLE || o.role.toUpperCase() === 'PUBLIC')
|
|
83
|
+
&& o.permissive && o.command === a.command && o.using === a.using && o.check === a.check);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function canonicalPolicyArms(contract: TableContract, governed?: ReadonlySet<string>): Array<[string, number]> {
|
|
87
|
+
const alive = contract.policies.flatMap(policyArms).filter((a) => !armDead(contract, a));
|
|
88
|
+
const kept = alive
|
|
89
|
+
.filter((a) => !armSubsumed(a, alive))
|
|
90
|
+
.filter((a) => (governed ? governedArm(governed, a.role) : true));
|
|
91
|
+
return multiset(kept.map(armKey));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function governedOnly<T>(map: Record<string, T> | undefined, governed: ReadonlySet<string> | undefined): Record<string, T> {
|
|
95
|
+
if (!map) return {};
|
|
96
|
+
if (!governed) return map;
|
|
97
|
+
return Object.fromEntries(
|
|
98
|
+
Object.entries(map).filter(([grantee]) => governed.has(grantee) || (grantee.toUpperCase() === 'PUBLIC' && governed.has('PUBLIC'))),
|
|
99
|
+
);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function canonicalAuthzV12(contract: TableContract, governed?: ReadonlySet<string>): Record<string, unknown> {
|
|
103
|
+
const grants = governedOnly(contract.grants, governed);
|
|
104
|
+
const columnGrants = governedOnly(contract.columnGrants, governed);
|
|
105
|
+
return {
|
|
106
|
+
table: contract.table,
|
|
107
|
+
rls: { enabled: contract.rls.enabled, forced: contract.rls.forced },
|
|
108
|
+
grants: Object.fromEntries(
|
|
109
|
+
Object.entries(grants).map(([role, privs]) => [role, [...(privs as string[])].sort()] as const).sort(([a], [b]) => (a < b ? -1 : 1)),
|
|
110
|
+
),
|
|
111
|
+
...(Object.keys(columnGrants).length > 0
|
|
112
|
+
? {
|
|
113
|
+
columnGrants: Object.fromEntries(
|
|
114
|
+
Object.entries(columnGrants)
|
|
115
|
+
.map(([role, byPriv]) => [
|
|
116
|
+
role,
|
|
117
|
+
Object.fromEntries(
|
|
118
|
+
Object.entries(byPriv as Record<string, string[]>).map(([priv, cols]) => [priv, [...cols].sort()] as const).sort(([a], [b]) => (a < b ? -1 : 1)),
|
|
119
|
+
),
|
|
120
|
+
] as const)
|
|
121
|
+
.sort(([a], [b]) => (a < b ? -1 : 1)),
|
|
122
|
+
),
|
|
123
|
+
}
|
|
124
|
+
: {}),
|
|
125
|
+
policies: canonicalPolicyArms(contract, governed),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// --- v12 base-schema canonical form (v13 adds the NOT VALID flag to the FK slice; everything
|
|
130
|
+
// else is identical, frozen here for self-containment) ---
|
|
131
|
+
|
|
132
|
+
function canonicalTableV12(table: TableSchema): Record<string, unknown> {
|
|
133
|
+
return {
|
|
134
|
+
table: table.table,
|
|
135
|
+
columns: byKey(
|
|
136
|
+
table.columns.map((c) => ({ name: c.name, type: c.type, notNull: c.notNull, default: normalizeDefault(c.default) })),
|
|
137
|
+
(c) => c.name,
|
|
138
|
+
),
|
|
139
|
+
primaryKey: table.primaryKey,
|
|
140
|
+
uniques: byKey(table.uniques.map((u) => ({ columns: u.columns })), (u) => u.columns.join(',')),
|
|
141
|
+
checks: byKey(table.checks.map((c) => ({ expr: normalizeCheck(c.expr) })), (c) => c.expr),
|
|
142
|
+
foreignKeys: byKey(
|
|
143
|
+
table.foreignKeys.map((f) => ({
|
|
144
|
+
columns: f.columns, refTable: f.refTable, refColumns: f.refColumns,
|
|
145
|
+
...(f.onDelete ? { onDelete: f.onDelete } : {}),
|
|
146
|
+
...(f.onUpdate ? { onUpdate: f.onUpdate } : {}),
|
|
147
|
+
})),
|
|
148
|
+
(f) => stableStringify(f),
|
|
149
|
+
),
|
|
150
|
+
indexes: [...new Set(table.indexes.map(indexKey))].sort(),
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// --- v12 serial-sequence ACL slice (frozen: USAGE only, exactly as v13 derives it) ---
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* The v12 inheritance rule: an INSERT-holder inherits USAGE on the table's serial sequence —
|
|
158
|
+
* exactly what `nextval` needs, and exactly what v13 still derives. Pinned as a literal here
|
|
159
|
+
* (rather than imported) so a LATER narrowing or widening of the live rule cannot move a hash
|
|
160
|
+
* this function is supposed to reproduce forever.
|
|
161
|
+
*/
|
|
162
|
+
function inheritedSequenceGrantsV12(table: TableContract): Record<string, string[]> {
|
|
163
|
+
const out: Record<string, string[]> = {};
|
|
164
|
+
for (const [grantee, privileges] of Object.entries(table.grants)) {
|
|
165
|
+
if (privileges.includes('INSERT')) out[grantee] = ['USAGE'];
|
|
166
|
+
}
|
|
167
|
+
return out;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function isGovernedGrantee(governed: ReadonlySet<string> | undefined, grantee: string): boolean {
|
|
171
|
+
if (!governed) return true;
|
|
172
|
+
return governed.has(grantee) || (grantee.toUpperCase() === 'PUBLIC' && governed.has('PUBLIC'));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function seqShape(grants: Record<string, string[]>, governed?: ReadonlySet<string>): string {
|
|
176
|
+
const kept = Object.entries(grants)
|
|
177
|
+
.filter(([grantee, privs]) => privs.length > 0 && isGovernedGrantee(governed, grantee))
|
|
178
|
+
.map(([grantee, privs]) => [grantee, [...privs].sort()] as const);
|
|
179
|
+
return stableStringify(Object.fromEntries(kept));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
function canonicalSequenceAuthzV12(
|
|
183
|
+
tables: TableSchema[],
|
|
184
|
+
authzTables: TableContract[],
|
|
185
|
+
measured: SequenceGrantContract[] | undefined,
|
|
186
|
+
governed: ReadonlySet<string> | undefined,
|
|
187
|
+
keepTable: (name: string) => boolean,
|
|
188
|
+
): Array<[string, string[]]> {
|
|
189
|
+
const snapshotByTable = new Map(tables.map((t) => [t.table, t]));
|
|
190
|
+
const measuredByTable = new Map<string, SequenceGrantContract[]>();
|
|
191
|
+
for (const s of measured ?? []) {
|
|
192
|
+
(measuredByTable.get(s.table) ?? measuredByTable.set(s.table, []).get(s.table)!).push(s);
|
|
193
|
+
}
|
|
194
|
+
const out: Array<[string, string[]]> = [];
|
|
195
|
+
for (const az of authzTables) {
|
|
196
|
+
if (!keepTable(az.table)) continue;
|
|
197
|
+
let shapes: string[];
|
|
198
|
+
if (measuredByTable.has(az.table)) {
|
|
199
|
+
shapes = measuredByTable.get(az.table)!.filter((s) => !s.identity).map((s) => seqShape(s.grants, governed));
|
|
200
|
+
} else {
|
|
201
|
+
const snap = snapshotByTable.get(az.table);
|
|
202
|
+
const serialCount = (snap?.columns ?? []).filter((c) => nextvalSequence(normalizeDefault(c.default)) != null).length;
|
|
203
|
+
if (serialCount === 0) continue;
|
|
204
|
+
const inherited = seqShape(inheritedSequenceGrantsV12(az), governed);
|
|
205
|
+
shapes = Array.from({ length: serialCount }, () => inherited);
|
|
206
|
+
}
|
|
207
|
+
if (shapes.length === 0) continue;
|
|
208
|
+
out.push([az.table, shapes.sort()]);
|
|
209
|
+
}
|
|
210
|
+
return out.sort(([a], [b]) => (a < b ? -1 : 1));
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** The frozen v12 live hash for (snapshot, authzTables, opts). */
|
|
214
|
+
export function fingerprintV12(snapshot: SchemaSnapshot, authzTables: TableContract[], opts: CanonicalizeOptions = {}): string {
|
|
215
|
+
const inScope = opts.schemas ? new Set(opts.schemas) : null;
|
|
216
|
+
const keepTable = (name: string) => inScope === null || inScope.has(schemaOf(name));
|
|
217
|
+
const tables = snapshot.tables.filter((t) => keepTable(t.table));
|
|
218
|
+
const usedEnum = new Set<string>();
|
|
219
|
+
for (const t of tables) for (const c of t.columns) usedEnum.add(schemaOf(c.type) === 'public' ? c.type : c.type.slice(c.type.indexOf('.') + 1));
|
|
220
|
+
const sequences = byKey(
|
|
221
|
+
(snapshot.sequences ?? [])
|
|
222
|
+
.filter((s) => inScope === null || inScope.has(schemaOf(s.name)))
|
|
223
|
+
.map((s) => ({ name: s.name, as: s.as, ...(s.start != null ? { start: s.start } : {}), ...(s.increment != null ? { increment: s.increment } : {}) })),
|
|
224
|
+
(s) => String(s.name),
|
|
225
|
+
);
|
|
226
|
+
const sequenceAuthz = canonicalSequenceAuthzV12(tables, authzTables, opts.sequenceGrants, opts.governedRoles, keepTable);
|
|
227
|
+
const canonical = {
|
|
228
|
+
v: 12,
|
|
229
|
+
tables: byKey(tables.map(canonicalTableV12), (t) => String(t.table)),
|
|
230
|
+
enums: byKey(
|
|
231
|
+
(snapshot.enums ?? []).filter((e) => inScope === null || usedEnum.has(e.name)).map((e) => ({ name: e.name, values: e.values })),
|
|
232
|
+
(e) => e.name,
|
|
233
|
+
),
|
|
234
|
+
authz: byKey(authzTables.filter((t) => keepTable(t.table)).map((t) => canonicalAuthzV12(t, opts.governedRoles)), (t) => String(t.table)),
|
|
235
|
+
...(sequences.length ? { sequences } : {}),
|
|
236
|
+
...(sequenceAuthz.length ? { sequenceAuthz } : {}),
|
|
237
|
+
};
|
|
238
|
+
return createHash('sha256').update(stableStringify(canonical)).digest('hex');
|
|
239
|
+
}
|
|
@@ -61,6 +61,7 @@ import { governedRoleSet, inheritedSequenceGrants } from './authz-reconcile';
|
|
|
61
61
|
import { normalizeDefault, normalizeCheck, indexKey, nextvalSequence } from './schema-diff';
|
|
62
62
|
import { fingerprintV10 } from './schema-fingerprint-v10';
|
|
63
63
|
import { fingerprintV11 } from './schema-fingerprint-v11';
|
|
64
|
+
import { fingerprintV12 } from './schema-fingerprint-v12';
|
|
64
65
|
|
|
65
66
|
// v2: expression fields (defaults, checks, partial-index WHERE) normalize through
|
|
66
67
|
// the diff's pg-deparse normalizers before hashing, so the model form and the live
|
|
@@ -141,7 +142,20 @@ import { fingerprintV11 } from './schema-fingerprint-v11';
|
|
|
141
142
|
// On every deployed stage the next db:plan mints a REVOKE narrowing those sequence grants to USAGE
|
|
142
143
|
// (the report classifies it as narrowing access). The v11 canon is frozen in schema-fingerprint-v11
|
|
143
144
|
// and registered in RETAINED_CANONS so a schema_log row recorded under it still descends.
|
|
144
|
-
|
|
145
|
+
// v13 — a NOT VALID foreign key is its OWN state. A foreign key added `NOT VALID` exists and is
|
|
146
|
+
// enforced on every new write, but has never been checked against the rows already in the table:
|
|
147
|
+
// it is the declared key one step short, and that step is precisely where a two-step add
|
|
148
|
+
// (`ADD … NOT VALID` then `VALIDATE`) can be interrupted. The v12 form hashed a key's columns,
|
|
149
|
+
// target and referential actions only, so the half-applied key and the validated one the models
|
|
150
|
+
// declare were the SAME hash — `db:fingerprint` reported MATCH on a database whose rows no
|
|
151
|
+
// constraint had ever been proved against, and a re-plan had nothing to emit. The flag now rides
|
|
152
|
+
// the canonical form (written only when true, so a fully-valid database's hash is unmoved by the
|
|
153
|
+
// field itself; the `v` stamp moves it, like every bump). Same consequence as every bump: a plan
|
|
154
|
+
// or baseline minted under v12 refuses as a FORMAT CHANGE and must be re-minted (db:plan) or
|
|
155
|
+
// re-baselined (db:reconcile --rebaseline) — no DDL, the fingerprint is a content address. The
|
|
156
|
+
// v12 canon is frozen in schema-fingerprint-v12 and registered in RETAINED_CANONS so a schema_log
|
|
157
|
+
// row recorded under it still descends.
|
|
158
|
+
export const FINGERPRINT_VERSION = 13;
|
|
145
159
|
|
|
146
160
|
// ---------------------------------------------------------------------------
|
|
147
161
|
// Canonical form.
|
|
@@ -178,11 +192,18 @@ function canonicalTable(table: TableSchema): Record<string, unknown> {
|
|
|
178
192
|
// Predicates hash as the diff compares them — one layer of pg's wrapping parens dropped
|
|
179
193
|
// (`(scope = ANY (...))` == `scope = ANY (...)`), whitespace collapsed.
|
|
180
194
|
checks: byKey(table.checks.map((c) => ({ expr: normalizeCheck(c.expr) })), (c) => c.expr),
|
|
195
|
+
// v13: a key pg has NEVER CHECKED against the rows already in the table is not the key the
|
|
196
|
+
// models declare — it is that key one step short, and the difference is exactly the step a
|
|
197
|
+
// two-step add can be interrupted between. Without the flag a half-applied foreign key
|
|
198
|
+
// hashed identically to a validated one, so `db:fingerprint` reported MATCH on a database
|
|
199
|
+
// whose data no constraint had ever been proved against. Written only when true, so every
|
|
200
|
+
// fully-valid database keeps the hash it had.
|
|
181
201
|
foreignKeys: byKey(
|
|
182
202
|
table.foreignKeys.map((f) => ({
|
|
183
203
|
columns: f.columns, refTable: f.refTable, refColumns: f.refColumns,
|
|
184
204
|
...(f.onDelete ? { onDelete: f.onDelete } : {}),
|
|
185
205
|
...(f.onUpdate ? { onUpdate: f.onUpdate } : {}),
|
|
206
|
+
...(f.notValid ? { notValid: true } : {}),
|
|
186
207
|
})),
|
|
187
208
|
(f) => stableStringify(f),
|
|
188
209
|
),
|
|
@@ -500,13 +521,15 @@ export type CanonHasher = (
|
|
|
500
521
|
* absent from this map is unrecomputable: `fingerprintLiveAtVersion` returns null and descent
|
|
501
522
|
* SKIPS that row (fails closed), never guesses a hash.
|
|
502
523
|
*
|
|
503
|
-
* Retired entries: `fingerprintV10` (no command fan, direct-grant deadness, no sequence ACLs)
|
|
524
|
+
* Retired entries: `fingerprintV10` (no command fan, direct-grant deadness, no sequence ACLs),
|
|
504
525
|
* `fingerprintV11` (the command fan and sequence ACLs, but the pre-least-privilege derivation that
|
|
505
|
-
* gave an INSERT-holder SELECT + UPDATE + USAGE on its sequence)
|
|
526
|
+
* gave an INSERT-holder SELECT + UPDATE + USAGE on its sequence), and `fingerprintV12` (everything
|
|
527
|
+
* v13 has except the foreign-key validity flag, so a NOT VALID key hashed as a validated one).
|
|
506
528
|
*/
|
|
507
529
|
export const RETAINED_CANONS: Readonly<Record<number, CanonHasher>> = {
|
|
508
530
|
10: (snapshot, tables, opts) => fingerprintV10(snapshot, tables, opts),
|
|
509
531
|
11: (snapshot, tables, opts) => fingerprintV11(snapshot, tables, opts),
|
|
532
|
+
12: (snapshot, tables, opts) => fingerprintV12(snapshot, tables, opts),
|
|
510
533
|
[FINGERPRINT_VERSION]: (snapshot, tables, opts) => fingerprintState(snapshot, tables, opts).hash,
|
|
511
534
|
};
|
|
512
535
|
|
|
@@ -52,6 +52,18 @@ export interface ForeignKey {
|
|
|
52
52
|
/** Lowercased referential actions, omitted when the Postgres default (`no action`). */
|
|
53
53
|
onDelete?: string;
|
|
54
54
|
onUpdate?: string;
|
|
55
|
+
/**
|
|
56
|
+
* The constraint exists but has NOT been checked against the rows already in the table
|
|
57
|
+
* (`convalidated = false`; `pg_get_constraintdef` spells it `… NOT VALID`).
|
|
58
|
+
*
|
|
59
|
+
* LIVE-SIDE ONLY. A Model declares a foreign key, never a half-installed one, so the
|
|
60
|
+
* compiled side never sets this — which is exactly what makes it visible as drift: a live
|
|
61
|
+
* NOT VALID key is a DIFFERENT state from the declared (validated) one. It rides the
|
|
62
|
+
* canonical fingerprint form for the same reason, so a half-applied key can never report
|
|
63
|
+
* MATCH. Omitted (not `false`) when the constraint is validated, so every already-recorded
|
|
64
|
+
* hash for a fully-valid database is untouched.
|
|
65
|
+
*/
|
|
66
|
+
notValid?: boolean;
|
|
55
67
|
}
|
|
56
68
|
|
|
57
69
|
export interface UniqueConstraint {
|
|
@@ -416,6 +428,8 @@ export interface ConstraintDescriptor {
|
|
|
416
428
|
/** FK referential actions, lowercased; omitted when the Postgres default (`no action`). */
|
|
417
429
|
onDelete?: string;
|
|
418
430
|
onUpdate?: string;
|
|
431
|
+
/** FK installed but never checked against the existing rows (`… NOT VALID`); omitted when valid. */
|
|
432
|
+
notValid?: boolean;
|
|
419
433
|
}
|
|
420
434
|
|
|
421
435
|
/** Parse one `pg_get_constraintdef` row into a structured descriptor. */
|
|
@@ -448,6 +462,12 @@ export function constraintRowToDescriptor(row: ConstraintRow): ConstraintDescrip
|
|
|
448
462
|
refColumns: m ? parseColumnList(m[3]) : [],
|
|
449
463
|
onDelete: action('DELETE'),
|
|
450
464
|
onUpdate: action('UPDATE'),
|
|
465
|
+
// `pg_get_constraintdef` appends NOT VALID to a constraint pg has never checked against
|
|
466
|
+
// the rows already in the table — the state a two-step FK add leaves behind between its
|
|
467
|
+
// ADD and its VALIDATE. Read from the SAME deparsed text as everything else here, so no
|
|
468
|
+
// second catalog column has to be threaded through every caller of CONSTRAINTS_SQL.
|
|
469
|
+
// Omitted when valid: absence is what keeps a fully-valid snapshot byte-identical.
|
|
470
|
+
...(/\bNOT\s+VALID\s*$/i.test(def.trim()) ? { notValid: true } : {}),
|
|
451
471
|
};
|
|
452
472
|
}
|
|
453
473
|
// check — keep the predicate text inside the outer CHECK ( ... ), normalized.
|
|
@@ -662,6 +682,9 @@ export function assembleSchema(rows: SchemaRows): SchemaSnapshot {
|
|
|
662
682
|
name: d.name, columns: d.columns ?? [], refTable: d.refTable ?? '', refColumns: d.refColumns ?? [],
|
|
663
683
|
...(d.onDelete ? { onDelete: d.onDelete } : {}),
|
|
664
684
|
...(d.onUpdate ? { onUpdate: d.onUpdate } : {}),
|
|
685
|
+
// Carried, not dropped: an unchecked key is a DIFFERENT state from a checked one, and
|
|
686
|
+
// the snapshot is what the differ and the fingerprint both read.
|
|
687
|
+
...(d.notValid ? { notValid: true } : {}),
|
|
665
688
|
});
|
|
666
689
|
break;
|
|
667
690
|
}
|
package/src/cli/state-apply.ts
CHANGED
|
@@ -26,7 +26,12 @@ import { FUNCTIONS_SQL, contractFunctionRow } from './security-catalog';
|
|
|
26
26
|
import { generateMigrationSql, HELD_DROP_PREFIX } from './migration-generate';
|
|
27
27
|
import type { SequenceDescriptor } from '@everystack/model';
|
|
28
28
|
import { fingerprintLive, FINGERPRINT_VERSION } from './schema-fingerprint';
|
|
29
|
-
import {
|
|
29
|
+
import {
|
|
30
|
+
ENSURE_RECONCILER_SQL,
|
|
31
|
+
renderSchemaLogInsert,
|
|
32
|
+
renderSchemaLogFingerprintUpdate,
|
|
33
|
+
renderSchemaLogCompletionUpdate,
|
|
34
|
+
} from './derived-apply';
|
|
30
35
|
import { isPureComment, statementBody, statementHead } from './sql-statement';
|
|
31
36
|
|
|
32
37
|
const NOTICE_PREFIX = '-- NOTICE';
|
|
@@ -52,6 +57,32 @@ export function classifyGeneratedStatements(statements: string[]): ClassifiedSta
|
|
|
52
57
|
return { executable, heldDrops, notices };
|
|
53
58
|
}
|
|
54
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Split an executable stream into the EDGE and the CHECKS: `ALTER TABLE … VALIDATE CONSTRAINT`
|
|
62
|
+
* is held out of the batch and run afterwards, one statement at a time.
|
|
63
|
+
*
|
|
64
|
+
* This is what makes the two-step foreign-key add worth splitting at all. `ADD CONSTRAINT …
|
|
65
|
+
* NOT VALID` holds an ACCESS EXCLUSIVE lock until its transaction COMMITS, so a VALIDATE in the
|
|
66
|
+
* same transaction runs under that lock and blocks every writer for the whole scan — precisely
|
|
67
|
+
* the outage the split exists to avoid. Run after the commit, the VALIDATE takes its own, much
|
|
68
|
+
* weaker SHARE UPDATE EXCLUSIVE lock, which readers and writers pass through.
|
|
69
|
+
*
|
|
70
|
+
* The second consequence is durability, and it is the point of doing it here rather than in the
|
|
71
|
+
* emitter: a VALIDATE that fails no longer rolls back the edge. The key stays installed and
|
|
72
|
+
* NOT VALID — enforced on every new write, unproven against the old rows — which is a state the
|
|
73
|
+
* next `db:plan` sees and finishes with a VALIDATE alone. An edge that rolled back instead would
|
|
74
|
+
* have thrown away a correct, committed schema change because of data it does not own.
|
|
75
|
+
*/
|
|
76
|
+
export function splitValidations(executable: readonly string[]): { batch: string[]; validations: string[] } {
|
|
77
|
+
const batch: string[] = [];
|
|
78
|
+
const validations: string[] = [];
|
|
79
|
+
for (const statement of executable) {
|
|
80
|
+
if (/^ALTER\s+TABLE\s+[\s\S]+?\sVALIDATE\s+CONSTRAINT\b/i.test(statementBody(statement))) validations.push(statement);
|
|
81
|
+
else batch.push(statement);
|
|
82
|
+
}
|
|
83
|
+
return { batch, validations };
|
|
84
|
+
}
|
|
85
|
+
|
|
55
86
|
/**
|
|
56
87
|
* The destructive subset of an executable statement stream — destructive
|
|
57
88
|
* means DATA LOSS (brick 9's classification, design decision 11):
|
|
@@ -105,12 +136,14 @@ const ADDITIVE_MATCHERS: RegExp[] = [
|
|
|
105
136
|
/^GRANT\b/,
|
|
106
137
|
/^COMMENT\s+ON\b/,
|
|
107
138
|
// ADD COLUMN alone. Its siblings `ADD CONSTRAINT` (FOREIGN KEY / CHECK / UNIQUE) and
|
|
108
|
-
// `ADD PRIMARY KEY` are deliberately NOT here: each one
|
|
109
|
-
//
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
//
|
|
113
|
-
//
|
|
139
|
+
// `ADD PRIMARY KEY` are deliberately NOT here: each one RESTRICTS every write after it, and
|
|
140
|
+
// all but the NOT VALID form also VALIDATE every row already in the table. A CHECK add fails
|
|
141
|
+
// on any violating row, a UNIQUE add fails on existing duplicates, and ADD PRIMARY KEY fails
|
|
142
|
+
// on duplicates or NULLs *and* silently makes every included column NOT NULL. A foreign key
|
|
143
|
+
// is emitted `NOT VALID` and so cannot fail on the old rows — but its `VALIDATE CONSTRAINT`
|
|
144
|
+
// can, and it still refuses every future write that breaks it. Calling any of these additive
|
|
145
|
+
// tells the reviewer the plan is safe when it can abort the apply outright — the same false
|
|
146
|
+
// zero described above. They read as `unclassified`, like `SET NOT NULL`.
|
|
114
147
|
/^ALTER\s+TABLE\s+[\s\S]+?\sADD\s+COLUMN\b/,
|
|
115
148
|
// Enabling/forcing RLS only ever RESTRICTS; it cannot widen access or lose a row.
|
|
116
149
|
/^ALTER\s+TABLE\s+[\s\S]+?\s(ENABLE|FORCE)\s+ROW\s+LEVEL\s+SECURITY\b/,
|
|
@@ -344,6 +377,30 @@ export interface StateApplyOptions {
|
|
|
344
377
|
* row is still a witness.
|
|
345
378
|
*/
|
|
346
379
|
descentNote?: () => string | undefined;
|
|
380
|
+
/**
|
|
381
|
+
* Read the live base fingerprint, for the row written the instant the batch commits.
|
|
382
|
+
*
|
|
383
|
+
* The `state apply` row is only a descent WITNESS if it carries a `to_fingerprint`
|
|
384
|
+
* (SCHEMA_LOG_WITNESS_SQL requires one), and it must be the hash of the database AS IT IS at
|
|
385
|
+
* that moment — with the new key installed and NOT VALID. Stamping the plan's target state
|
|
386
|
+
* there would be a prediction, not a record: if the process then dies mid-VALIDATE, the next
|
|
387
|
+
* apply recomputes the live hash, does not match the stamped one, and the row it was given to
|
|
388
|
+
* protect itself with is inert. The plan's own endpoint is stamped by the caller AFTER the
|
|
389
|
+
* last check passes.
|
|
390
|
+
*
|
|
391
|
+
* Optional: without it the row still goes in before the checks, and the caller fills the
|
|
392
|
+
* fingerprint in when it re-introspects (db:sync's lane). Called ONLY when there are checks to
|
|
393
|
+
* run — with none there is no window between the commit and the record to protect.
|
|
394
|
+
*/
|
|
395
|
+
readFingerprint?: () => Promise<string>;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/** A `VALIDATE CONSTRAINT` that did not pass — the key is installed but still unproven. */
|
|
399
|
+
export interface PendingValidation {
|
|
400
|
+
/** The exact statement to re-run once the data satisfies the key. */
|
|
401
|
+
sql: string;
|
|
402
|
+
/** PostgreSQL's message, trimmed for the record. */
|
|
403
|
+
error: string;
|
|
347
404
|
}
|
|
348
405
|
|
|
349
406
|
export interface StateApplyResult {
|
|
@@ -354,12 +411,25 @@ export interface StateApplyResult {
|
|
|
354
411
|
batchSql: string;
|
|
355
412
|
/** schema_log row id for this apply, when one was written. */
|
|
356
413
|
logId?: number;
|
|
414
|
+
/**
|
|
415
|
+
* Constraints left NOT VALID because their post-commit `VALIDATE` failed. The edge itself
|
|
416
|
+
* COMMITTED — this is not a rollback and must never be reported as one. Empty on a clean apply.
|
|
417
|
+
*/
|
|
418
|
+
pendingValidations: PendingValidation[];
|
|
357
419
|
}
|
|
358
420
|
|
|
359
421
|
/**
|
|
360
422
|
* Execute the executable subset of a generated diff as one batch and record
|
|
361
423
|
* it in schema_log. A failed batch rolls back (one implicit transaction),
|
|
362
424
|
* logs the failure best-effort, and rethrows.
|
|
425
|
+
*
|
|
426
|
+
* ORDER OF WRITES, and it is the point: batch → RECORD → checks → amend the record. The
|
|
427
|
+
* `VALIDATE CONSTRAINT` pass is the longest thing an apply does (a full scan per key) and the
|
|
428
|
+
* one a wall clock or a dropped connection actually kills. Recording after it meant that death
|
|
429
|
+
* left the DDL committed with nothing on the record — and a live state no commit on any ref
|
|
430
|
+
* declares reads to the NEXT apply as proven drift, which is the loudest possible wrong answer.
|
|
431
|
+
* So the row is written as soon as the batch has committed, describing exactly what is true then
|
|
432
|
+
* (the key installed, none of the checks run yet), and amended when the checks are over.
|
|
363
433
|
*/
|
|
364
434
|
export async function applyGeneratedStatements(
|
|
365
435
|
runner: QueryRunner,
|
|
@@ -368,18 +438,28 @@ export async function applyGeneratedStatements(
|
|
|
368
438
|
): Promise<StateApplyResult> {
|
|
369
439
|
const classified = classifyGeneratedStatements(statements);
|
|
370
440
|
if (classified.executable.length === 0) {
|
|
371
|
-
return { applied: false, classified, batchSql: '' };
|
|
441
|
+
return { applied: false, classified, batchSql: '', pendingValidations: [] };
|
|
372
442
|
}
|
|
373
443
|
|
|
374
444
|
const now = options.now ?? Date.now;
|
|
375
445
|
await runner(ENSURE_RECONCILER_SQL.join(';\n'));
|
|
376
446
|
|
|
447
|
+
// The row checks run AFTER the edge commits — see splitValidations for why that is the whole
|
|
448
|
+
// point of the two-step add, and why a failed check is a pending state rather than a rollback.
|
|
449
|
+
const { batch, validations } = splitValidations(classified.executable);
|
|
450
|
+
|
|
377
451
|
// Semicolon on its own line: a statement whose last line is a comment
|
|
378
452
|
// cannot swallow the separator.
|
|
379
|
-
|
|
453
|
+
//
|
|
454
|
+
// `batchSql` is the batch that RAN IN THIS TRANSACTION — the checks are not in it. The record
|
|
455
|
+
// written below is a record of what committed, and the checks had not been attempted at that
|
|
456
|
+
// point; they join the row on the amendment, where their outcome is known.
|
|
457
|
+
const batchSql = batch.join('\n;\n');
|
|
458
|
+
const validationSql = validations.join('\n;\n');
|
|
459
|
+
const fullSql = [batchSql, validationSql].filter((s) => s.length > 0).join('\n;\n');
|
|
380
460
|
const started = now();
|
|
381
461
|
try {
|
|
382
|
-
await runner(batchSql);
|
|
462
|
+
if (batch.length > 0) await runner(batchSql);
|
|
383
463
|
} catch (err: any) {
|
|
384
464
|
try {
|
|
385
465
|
await runner(renderSchemaLogInsert({
|
|
@@ -395,16 +475,79 @@ export async function applyGeneratedStatements(
|
|
|
395
475
|
}
|
|
396
476
|
|
|
397
477
|
const note = options.descentNote?.();
|
|
478
|
+
// The edge APPLIED — that word is load-bearing (the descent ledger matches `applied%`, and the
|
|
479
|
+
// schema changes really are committed). What is still unproven is named in the same line, so
|
|
480
|
+
// nobody reads a half-proven key as a finished one.
|
|
481
|
+
const outcomeWith = (pending: number): string =>
|
|
482
|
+
['applied', note, pending > 0 ? `${pending} constraint(s) PENDING validation` : undefined]
|
|
483
|
+
.filter(Boolean)
|
|
484
|
+
.join(' — ');
|
|
485
|
+
|
|
486
|
+
// The hash of what just committed, read now rather than predicted. Only when checks follow:
|
|
487
|
+
// with none, the record goes in the same breath as the commit (as it always has) and the
|
|
488
|
+
// caller's own post-apply introspection stamps it — reading it twice would buy nothing and
|
|
489
|
+
// cost a full introspection on every apply. Best-effort: a fingerprint that cannot be read
|
|
490
|
+
// must not cost us the record.
|
|
491
|
+
let landedAfterBatch: string | undefined;
|
|
492
|
+
if (validations.length > 0 && options.readFingerprint) {
|
|
493
|
+
try {
|
|
494
|
+
landedAfterBatch = await options.readFingerprint();
|
|
495
|
+
} catch { /* the record matters more than the stamp */ }
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
// THE WITNESS, written before the long step. Every check is still outstanding here, and the
|
|
499
|
+
// outcome says exactly that — this row is true at the instant it is written, not a prediction
|
|
500
|
+
// of how the checks will go.
|
|
398
501
|
const logRows = await runner(renderSchemaLogInsert({
|
|
399
|
-
kind: 'state apply', sql: batchSql, outcome:
|
|
502
|
+
kind: 'state apply', sql: batchSql, outcome: outcomeWith(validations.length),
|
|
400
503
|
actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
|
|
401
504
|
fromFingerprint: options.fromFingerprint,
|
|
505
|
+
...(landedAfterBatch ? { toFingerprint: landedAfterBatch } : {}),
|
|
402
506
|
fpVersion: options.fpVersion ?? FINGERPRINT_VERSION, dirty: options.dirty ?? false,
|
|
403
507
|
durationMs: now() - started,
|
|
404
508
|
}));
|
|
405
509
|
const logId = logRows?.[0]?.id !== undefined ? Number(logRows[0].id) : undefined;
|
|
406
510
|
|
|
407
|
-
|
|
511
|
+
// Every check is attempted, even after one fails: the operator is owed the COMPLETE list of
|
|
512
|
+
// keys still unproven, not the first one. Each runs on its own, so one failure cannot take a
|
|
513
|
+
// sibling's successful validation down with it.
|
|
514
|
+
const pendingValidations: PendingValidation[] = [];
|
|
515
|
+
for (const statement of validations) {
|
|
516
|
+
try {
|
|
517
|
+
await runner(statement);
|
|
518
|
+
} catch (err: any) {
|
|
519
|
+
pendingValidations.push({ sql: statement, error: String(err?.message ?? err).slice(0, 500) });
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
// The amendment: the SAME row, finished. A second `state apply` row would put two applies in
|
|
524
|
+
// the ledger for one edge — and the descent witness reads that ledger. So the checks and their
|
|
525
|
+
// result land on the row that already describes this apply, never beside it.
|
|
526
|
+
if (validations.length > 0 && logId !== undefined) {
|
|
527
|
+
try {
|
|
528
|
+
await runner(renderSchemaLogCompletionUpdate(logId, {
|
|
529
|
+
outcome: outcomeWith(pendingValidations.length),
|
|
530
|
+
sql: fullSql,
|
|
531
|
+
durationMs: now() - started,
|
|
532
|
+
}));
|
|
533
|
+
} catch { /* the memoir is best-effort; the row already says what committed */ }
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
// One row per unproven key, carrying the exact statement to re-run. Best-effort, like every
|
|
537
|
+
// other memoir write: failing to record why a key is pending must not mask that it is.
|
|
538
|
+
for (const pending of pendingValidations) {
|
|
539
|
+
try {
|
|
540
|
+
await runner(renderSchemaLogInsert({
|
|
541
|
+
kind: 'state validate', sql: pending.sql,
|
|
542
|
+
outcome: `pending: ${pending.error}`,
|
|
543
|
+
actor: options.actor, gitRef: options.gitRef, planRef: options.planRef,
|
|
544
|
+
fromFingerprint: options.fromFingerprint,
|
|
545
|
+
fpVersion: options.fpVersion ?? FINGERPRINT_VERSION, dirty: options.dirty ?? false,
|
|
546
|
+
}));
|
|
547
|
+
} catch { /* see above */ }
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
return { applied: true, classified, batchSql, pendingValidations, ...(logId !== undefined ? { logId } : {}) };
|
|
408
551
|
}
|
|
409
552
|
|
|
410
553
|
// ---------------------------------------------------------------------------
|
|
@@ -433,6 +576,9 @@ export interface StateSyncOutcome {
|
|
|
433
576
|
toFingerprint: string;
|
|
434
577
|
/** Executable statements still differing after apply — empty is the bar. */
|
|
435
578
|
remaining: string[];
|
|
579
|
+
/** Constraints whose post-commit `VALIDATE` failed: installed, enforced, still unproven. They
|
|
580
|
+
* also show up in `remaining` as a VALIDATE the next run re-emits. */
|
|
581
|
+
pendingValidations: PendingValidation[];
|
|
436
582
|
/** schema_log row id, when DDL ran. */
|
|
437
583
|
logId?: number;
|
|
438
584
|
/** Fresh post-apply introspection (the pre-apply one when nothing ran). */
|
|
@@ -463,7 +609,7 @@ export async function applyStateAndVerify(
|
|
|
463
609
|
if (classified.executable.length === 0) {
|
|
464
610
|
return {
|
|
465
611
|
statements, classified, applied: false,
|
|
466
|
-
fromFingerprint, toFingerprint: fromFingerprint, remaining: [],
|
|
612
|
+
fromFingerprint, toFingerprint: fromFingerprint, remaining: [], pendingValidations: [],
|
|
467
613
|
after: { snapshot: current, contract: liveAuthz },
|
|
468
614
|
};
|
|
469
615
|
}
|
|
@@ -484,6 +630,7 @@ export async function applyStateAndVerify(
|
|
|
484
630
|
return {
|
|
485
631
|
statements, classified, applied: true,
|
|
486
632
|
fromFingerprint, toFingerprint, remaining,
|
|
633
|
+
pendingValidations: result.pendingValidations,
|
|
487
634
|
...(result.logId !== undefined ? { logId: result.logId } : {}),
|
|
488
635
|
after: { snapshot, contract },
|
|
489
636
|
};
|