@everystack/cli 0.4.45 → 0.4.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cli/alter-type-dependents.ts +96 -0
- package/src/cli/apply-execute.ts +22 -8
- package/src/cli/authz-adoption-class.ts +75 -26
- package/src/cli/authz-canonical.ts +37 -5
- package/src/cli/authz-compile.ts +87 -37
- package/src/cli/authz-contract.ts +92 -19
- package/src/cli/authz-derive.ts +158 -33
- package/src/cli/authz-reconcile.ts +48 -6
- package/src/cli/aws.ts +32 -0
- package/src/cli/commands/db-apply.ts +82 -20
- package/src/cli/commands/db-authz.ts +9 -14
- package/src/cli/commands/db-backfill.ts +1 -1
- package/src/cli/commands/db-exec.ts +20 -1
- package/src/cli/commands/db-fingerprint.ts +54 -18
- package/src/cli/commands/db-generate.ts +11 -17
- package/src/cli/commands/db-plan.ts +89 -9
- package/src/cli/commands/db-pull.ts +16 -18
- package/src/cli/commands/db-reconcile.ts +19 -21
- package/src/cli/commands/db-refresh.ts +33 -5
- package/src/cli/commands/db-swap.ts +5 -4
- package/src/cli/commands/db-sync.ts +8 -5
- package/src/cli/commands/db.ts +2 -1
- package/src/cli/db-build.ts +2 -2
- package/src/cli/db-source.ts +56 -0
- package/src/cli/derived-introspect.ts +27 -26
- package/src/cli/derived-lint.ts +7 -8
- package/src/cli/edge-plan.ts +112 -16
- package/src/cli/exec-digest.ts +55 -13
- package/src/cli/git-descent.ts +16 -9
- package/src/cli/index.ts +3 -3
- package/src/cli/model-render.ts +56 -50
- package/src/cli/schema-compile.ts +6 -1
- package/src/cli/schema-diff.ts +1 -1
- package/src/cli/schema-fingerprint.ts +67 -7
- package/src/cli/schema-introspect.ts +44 -17
- package/src/cli/schema-source.ts +9 -0
- package/src/cli/session.ts +184 -0
- package/src/cli/stage-read-consistency.ts +145 -0
- package/src/cli/state-apply.ts +4 -2
- package/src/cli/swap-execute.ts +4 -3
- package/src/cli/search-path.ts +0 -51
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
import { createHash } from 'node:crypto';
|
|
21
21
|
import { IGNORED_SCHEMAS, type QueryRunner } from './authz-contract.js';
|
|
22
22
|
import { normalizeSql, type DerivedKind } from './derived-source.js';
|
|
23
|
-
import {
|
|
23
|
+
import { INTROSPECTION_SESSION, rowsOrEmpty, type SessionRunner } from './session.js';
|
|
24
24
|
|
|
25
25
|
export interface LiveObject {
|
|
26
26
|
kind: DerivedKind;
|
|
@@ -591,30 +591,31 @@ export function assembleDerivedCatalog(rows: DerivedRows): DerivedCatalog {
|
|
|
591
591
|
* injected runner; a missing provenance table (first run against a database
|
|
592
592
|
* the reconciler has never touched) folds to an empty claims list.
|
|
593
593
|
*/
|
|
594
|
-
export async function introspectDerived(
|
|
595
|
-
//
|
|
596
|
-
// renders a ref bare while its schema is in scope, qualified once off)
|
|
597
|
-
//
|
|
598
|
-
//
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
594
|
+
export async function introspectDerived(session: SessionRunner): Promise<DerivedCatalog> {
|
|
595
|
+
// ONE session: view/matview/function bodies deparse relative to the search_path
|
|
596
|
+
// (`pg_get_viewdef` renders a ref bare while its schema is in scope, qualified once off),
|
|
597
|
+
// so the recorded def hash is only stable if every body is captured under one pinned path
|
|
598
|
+
// at one moment.
|
|
599
|
+
//
|
|
600
|
+
// The provenance read is `allowFailure`: `everystack.derived_provenance` does not exist
|
|
601
|
+
// until something has been reconciled, and its absence is an ANSWER (no claims), not a
|
|
602
|
+
// failure. Before the session it was a try/catch, which is the same intent expressed the
|
|
603
|
+
// only way a one-statement-at-a-time runner could — by being willing to lose the session.
|
|
604
|
+
const [relations, functions, indexes, depends, triggers, grants, provenance] = await session(
|
|
605
|
+
[
|
|
606
|
+
DERIVED_RELATIONS_SQL, DERIVED_FUNCTIONS_SQL, MATVIEW_INDEXES_SQL, DERIVED_DEPENDS_SQL,
|
|
607
|
+
DERIVED_TRIGGERS_SQL, DERIVED_GRANTS_SQL,
|
|
608
|
+
{ sql: PROVENANCE_SQL, allowFailure: true },
|
|
609
|
+
],
|
|
610
|
+
INTROSPECTION_SESSION,
|
|
611
|
+
);
|
|
612
|
+
return assembleDerivedCatalog({
|
|
613
|
+
relations: relations as RelationRow[],
|
|
614
|
+
functions: functions as FunctionRow[],
|
|
615
|
+
indexes: indexes as IndexDefRow[],
|
|
616
|
+
depends: depends as DependsRow[],
|
|
617
|
+
triggers: triggers as TriggerRow[],
|
|
618
|
+
grants: grants as GrantAclRow[],
|
|
619
|
+
provenance: rowsOrEmpty(provenance) as ProvenanceRawRow[],
|
|
619
620
|
});
|
|
620
621
|
}
|
package/src/cli/derived-lint.ts
CHANGED
|
@@ -26,9 +26,10 @@
|
|
|
26
26
|
* grant.
|
|
27
27
|
*/
|
|
28
28
|
|
|
29
|
+
import { isColumnAbility } from '@everystack/model';
|
|
29
30
|
import type {
|
|
30
31
|
ModelDescriptor, DerivedDescriptor, ViewDescriptor, MaterializedViewDescriptor,
|
|
31
|
-
FunctionDescriptor, DependsOnRef,
|
|
32
|
+
FunctionDescriptor, DependsOnRef,
|
|
32
33
|
} from '@everystack/model';
|
|
33
34
|
import { parseQualified } from './derived-source.js';
|
|
34
35
|
|
|
@@ -47,11 +48,6 @@ function isModelRef(ref: DependsOnRef): ref is ModelDescriptor {
|
|
|
47
48
|
return 'table' in ref;
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
/** The column-scoped self read — same predicate as the table grant compiler. */
|
|
51
|
-
function isColumnRead(a: Ability): boolean {
|
|
52
|
-
return a.action === 'read' && Boolean(a.condition.owner) && Boolean(a.condition.columns?.length);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
51
|
/**
|
|
56
52
|
* Does the table's grant compile give `role` SELECT (full or column-scoped)?
|
|
57
53
|
*
|
|
@@ -65,8 +61,11 @@ function isColumnRead(a: Ability): boolean {
|
|
|
65
61
|
*/
|
|
66
62
|
export function tableReaches(m: ModelDescriptor, role: string): boolean {
|
|
67
63
|
for (const a of m.abilities) {
|
|
68
|
-
if (
|
|
69
|
-
|
|
64
|
+
if (isColumnAbility(a)) {
|
|
65
|
+
// Only the READ half reaches: a column-scoped update compiles to `UPDATE (cols)`
|
|
66
|
+
// and no SELECT, so counting it as reach would pass an invoker view the database
|
|
67
|
+
// then refuses at runtime — the false-negative this gate's docstring warns about.
|
|
68
|
+
if (a.action === 'read' && (a.condition.role ?? 'authenticated') === role) return true;
|
|
70
69
|
continue;
|
|
71
70
|
}
|
|
72
71
|
if (a.action !== 'read' && a.action !== 'manage') continue;
|
package/src/cli/edge-plan.ts
CHANGED
|
@@ -31,7 +31,7 @@ import type { AuthzContract } from './authz-contract.js';
|
|
|
31
31
|
import { generateMigrationSql, unmodeledTables } from './migration-generate.js';
|
|
32
32
|
import { compileTableContract } from './authz-compile.js';
|
|
33
33
|
import { classifyGeneratedStatements, classifyDestructive, partitionStatements, renderStatementHistogram } from './state-apply.js';
|
|
34
|
-
import { fingerprintLive, fingerprintState, fingerprintModels, stableStringify, compareStoredFingerprint, formatChangedMessage, FINGERPRINT_VERSION } from './schema-fingerprint.js';
|
|
34
|
+
import { fingerprintLive, fingerprintState, fingerprintModels, governedRolesForModels, stableStringify, compareStoredFingerprint, formatChangedMessage, FINGERPRINT_VERSION } from './schema-fingerprint.js';
|
|
35
35
|
import { compileDeclaredState } from './declared-diff.js';
|
|
36
36
|
import { compileTableRenames, compileTableMoves } from './schema-compile.js';
|
|
37
37
|
|
|
@@ -67,8 +67,21 @@ export interface EdgePlan {
|
|
|
67
67
|
* drift that no edit could explain. Absent on plans minted before v4.
|
|
68
68
|
*/
|
|
69
69
|
fpVersion?: number;
|
|
70
|
-
/**
|
|
70
|
+
/**
|
|
71
|
+
* The predicted live fingerprint after apply — exact, unmodeled-aware, and
|
|
72
|
+
* GOVERNED: hashed through the models' governed-role set (`governedRoles`
|
|
73
|
+
* below), because it is a declared-vs-live claim. Compare it only against
|
|
74
|
+
* `governedLiveFingerprint` computed with the same set — never against the
|
|
75
|
+
* raw `from` flavor.
|
|
76
|
+
*/
|
|
71
77
|
toFingerprint: string;
|
|
78
|
+
/**
|
|
79
|
+
* The governed-role set `to` was hashed under, sorted — carried on the plan
|
|
80
|
+
* so every apply venue verifies with the exact set the mint used. Absent on
|
|
81
|
+
* plans minted before this field existed (those predate the governed `to`
|
|
82
|
+
* and verify raw, as they were minted).
|
|
83
|
+
*/
|
|
84
|
+
governedRoles?: string[];
|
|
72
85
|
/** The models-only fingerprint — context for the strong claim. Equals `to` when nothing is unmodeled. */
|
|
73
86
|
declaredFingerprint: string;
|
|
74
87
|
/** The edge, in db:generate's statement grammar (no held drops — mint refuses them). */
|
|
@@ -78,6 +91,21 @@ export interface EdgePlan {
|
|
|
78
91
|
destructive: number;
|
|
79
92
|
classification: PlanClassification;
|
|
80
93
|
notices: number;
|
|
94
|
+
/**
|
|
95
|
+
* WHY each authorization statement exists, one entry per statement — the per-statement half
|
|
96
|
+
* of the aggregate the summary prints.
|
|
97
|
+
*
|
|
98
|
+
* Optional because it is re-derived against LIVE at mint time, so a plan minted without a
|
|
99
|
+
* reachable database (or before this field existed) simply has none. Never trusted as a
|
|
100
|
+
* stored claim: like the counts, it describes the target as it was when the plan was cut.
|
|
101
|
+
*/
|
|
102
|
+
adoption?: { table: string; subject: string; cls: string; why: string }[];
|
|
103
|
+
/**
|
|
104
|
+
* Tables this plan leaves with no surviving read path — resolved at mint against the LIVE
|
|
105
|
+
* contract, which is the only moment both halves of that question are in hand. Absent on
|
|
106
|
+
* plans minted before this field existed; the summary falls back to the text-only reading.
|
|
107
|
+
*/
|
|
108
|
+
dark?: string[];
|
|
81
109
|
/** Live tables no model declares — untouched by the edge, riding into `to` as-is. */
|
|
82
110
|
unmodeled: string[];
|
|
83
111
|
gitRef: string | null;
|
|
@@ -114,8 +142,14 @@ export function predictLiveFingerprint(
|
|
|
114
142
|
models: ModelDescriptor[],
|
|
115
143
|
snapshot: SchemaSnapshot,
|
|
116
144
|
contract: AuthzContract,
|
|
117
|
-
opts: { schema?: string } = {},
|
|
145
|
+
opts: { schema?: string; governedRoles?: ReadonlySet<string> } = {},
|
|
118
146
|
): string {
|
|
147
|
+
// The prediction is a DECLARED-vs-live comparison, so it must hash through the
|
|
148
|
+
// models' governed set — the live side of any comparison against it must use
|
|
149
|
+
// the same set (see governedLiveFingerprint). Unfiltered, every ride-through
|
|
150
|
+
// table's foreign grants poison the endpoint and verify-after can never pass
|
|
151
|
+
// on a brownfield target.
|
|
152
|
+
const governedRoles = opts.governedRoles ?? governedRolesForModels(models);
|
|
119
153
|
const declared = compileDeclaredState(models, { schema: opts.schema });
|
|
120
154
|
const declaredTables = new Set(declared.snapshot.tables.map((t) => t.table));
|
|
121
155
|
const currentNames = new Set(snapshot.tables.map((t) => t.table));
|
|
@@ -150,7 +184,21 @@ export function predictLiveFingerprint(
|
|
|
150
184
|
...declared.contract.tables,
|
|
151
185
|
...contract.tables.filter((t) => ridesThrough(t.table)),
|
|
152
186
|
];
|
|
153
|
-
return fingerprintState(mergedSnapshot, mergedAuthz).hash;
|
|
187
|
+
return fingerprintState(mergedSnapshot, mergedAuthz, { governedRoles }).hash;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* The live half of every declared-vs-live comparison: the target's state hashed
|
|
192
|
+
* through the SAME governed set as the prediction. `predicted === governedLive`
|
|
193
|
+
* is the declares-test; comparing a prediction against an UNFILTERED live hash
|
|
194
|
+
* is a category error that can never converge on a brownfield database.
|
|
195
|
+
*/
|
|
196
|
+
export function governedLiveFingerprint(
|
|
197
|
+
snapshot: SchemaSnapshot,
|
|
198
|
+
contract: AuthzContract,
|
|
199
|
+
governedRoles: ReadonlySet<string>,
|
|
200
|
+
): string {
|
|
201
|
+
return fingerprintState(snapshot, contract.tables, { governedRoles }).hash;
|
|
154
202
|
}
|
|
155
203
|
|
|
156
204
|
/**
|
|
@@ -193,12 +241,21 @@ export function mintEdgePlan(
|
|
|
193
241
|
const breakdown = partitionStatements(classified.executable, { declaredGrantees });
|
|
194
242
|
const destructive = breakdown.drops.length + breakdown.narrowings.length + breakdown.strips.length;
|
|
195
243
|
|
|
244
|
+
// The plan's two endpoints are two different FLAVORS, deliberately:
|
|
245
|
+
// - `from` is the RAW live hash (everything introspected) — the concurrency
|
|
246
|
+
// lock and the backup-stamp chain (backup.fp == plan.from) compare
|
|
247
|
+
// live-vs-live and must stay computable with no models in hand.
|
|
248
|
+
// - `to` is the GOVERNED hash — a declared-vs-live claim, filtered through
|
|
249
|
+
// the models' governed set, which rides ON the plan so every venue
|
|
250
|
+
// (direct, ops Lambda) verifies with the set the mint used.
|
|
251
|
+
const governedRoles = governedRolesForModels(models, opts.governedRoles);
|
|
196
252
|
return {
|
|
197
253
|
v: PLAN_VERSION,
|
|
198
254
|
fromFingerprint: fingerprintLive(snapshot, contract).hash,
|
|
199
255
|
fpVersion: FINGERPRINT_VERSION,
|
|
200
|
-
toFingerprint: predictLiveFingerprint(models, snapshot, contract, { schema: opts.schema }),
|
|
256
|
+
toFingerprint: predictLiveFingerprint(models, snapshot, contract, { schema: opts.schema, governedRoles }),
|
|
201
257
|
declaredFingerprint: fingerprintModels(models, { schema: opts.schema }).hash,
|
|
258
|
+
governedRoles: [...governedRoles].sort(),
|
|
202
259
|
statements,
|
|
203
260
|
executable: classified.executable.length,
|
|
204
261
|
destructive,
|
|
@@ -213,6 +270,10 @@ export function mintEdgePlan(
|
|
|
213
270
|
...(breakdown.unclassified.length > 0 ? { unclassified: breakdown.unclassified.length } : {}),
|
|
214
271
|
},
|
|
215
272
|
notices: classified.notices.length,
|
|
273
|
+
// Resolved HERE because this is the only place that holds both the edge and the live
|
|
274
|
+
// contract. The summary is rendered later from the plan alone, and answering "does a read
|
|
275
|
+
// survive?" from statement text is what produced three false positives on a real plan.
|
|
276
|
+
dark: tablesLeftWithoutARead(statements, contract),
|
|
216
277
|
unmodeled: unmodeledTables(models, snapshot),
|
|
217
278
|
gitRef: opts.gitRef ?? null,
|
|
218
279
|
mintedBy: opts.actor ?? null,
|
|
@@ -227,25 +288,58 @@ export function mintEdgePlan(
|
|
|
227
288
|
* enabled and the grant is still there, so the table returns ZERO rows to that role. On a real
|
|
228
289
|
* adoption plan that was 11 tables, including the users table, and it printed no notice at all.
|
|
229
290
|
*
|
|
230
|
-
* Deliberately conservative: it
|
|
231
|
-
*
|
|
232
|
-
*
|
|
291
|
+
* Deliberately conservative: it names a table only when NO read path survives the plan for any
|
|
292
|
+
* role, so it under-reports rather than crying wolf. Pass the LIVE contract to get that
|
|
293
|
+
* question answered properly — without it, a `DROP POLICY` statement does not say which
|
|
294
|
+
* command it policed and the fallback can only count drops.
|
|
233
295
|
*/
|
|
234
|
-
export function tablesLeftWithoutARead(statements: readonly string[]): string[] {
|
|
235
|
-
const
|
|
236
|
-
const
|
|
296
|
+
export function tablesLeftWithoutARead(statements: readonly string[], live?: AuthzContract): string[] {
|
|
297
|
+
const droppedByTable = new Map<string, Set<string>>();
|
|
298
|
+
const createdRead = new Set<string>();
|
|
299
|
+
const touched = new Set<string>();
|
|
300
|
+
|
|
237
301
|
for (const statement of statements) {
|
|
238
302
|
const head = (statement.split('\n').find((l) => l.trim() !== '' && !l.trim().startsWith('--')) ?? '').trim();
|
|
239
|
-
let m = /^DROP\s+POLICY\s+(?:IF\s+EXISTS\s+)
|
|
303
|
+
let m = /^DROP\s+POLICY\s+(?:IF\s+EXISTS\s+)?(\S+)\s+ON\s+(\S+?);?$/i.exec(head);
|
|
240
304
|
if (m) {
|
|
241
|
-
|
|
305
|
+
touched.add(m[2]);
|
|
306
|
+
const names = droppedByTable.get(m[2]) ?? new Set<string>();
|
|
307
|
+
names.add(m[1]);
|
|
308
|
+
droppedByTable.set(m[2], names);
|
|
242
309
|
continue;
|
|
243
310
|
}
|
|
244
311
|
m = /^CREATE\s+POLICY\s+\S+\s+ON\s+(\S+)/i.exec(head);
|
|
245
312
|
// Only a SELECT-admitting policy restores a read; an INSERT-only policy does not.
|
|
246
|
-
if (m && /\bFOR\s+(SELECT|ALL)\b/i.test(head))
|
|
313
|
+
if (m && /\bFOR\s+(SELECT|ALL)\b/i.test(head)) createdRead.add(m[1]);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// Without the live contract this can only count drops, and a DROP POLICY statement does not
|
|
317
|
+
// carry the command it policed. That is the old behaviour, kept for plans minted before the
|
|
318
|
+
// contract was threaded through: it OVER-reports, naming a table whose dropped policy was an
|
|
319
|
+
// UPDATE and whose reads were never touched.
|
|
320
|
+
if (!live) return [...droppedByTable.keys()].filter((t) => !createdRead.has(t)).sort();
|
|
321
|
+
|
|
322
|
+
// With it, ask the real question: does ANY policy admitting SELECT survive this plan? A
|
|
323
|
+
// table is dark only when none does — not merely because some policy on it was dropped.
|
|
324
|
+
//
|
|
325
|
+
// Three false positives on a real adoption plan came from the cheaper question: two tables
|
|
326
|
+
// lost only an UPDATE policy, and one lost a single role-scoped read while anon and
|
|
327
|
+
// authenticated reads survived untouched. A safety detector that cries wolf spends the
|
|
328
|
+
// credibility of its true positives, so this deliberately under-reports instead.
|
|
329
|
+
const byTable = new Map(live.tables.map((t) => [t.table, t]));
|
|
330
|
+
const dark: string[] = [];
|
|
331
|
+
for (const table of touched) {
|
|
332
|
+
if (createdRead.has(table)) continue;
|
|
333
|
+
const contract = byTable.get(table);
|
|
334
|
+
if (!contract) continue; // not a live table — nothing to go dark
|
|
335
|
+
if (!contract.rls?.enabled) continue; // RLS off: policies do not gate the read at all
|
|
336
|
+
const dropped = droppedByTable.get(table) ?? new Set<string>();
|
|
337
|
+
const readSurvives = contract.policies.some(
|
|
338
|
+
(p) => (p.command === 'SELECT' || p.command === 'ALL') && !dropped.has(p.name),
|
|
339
|
+
);
|
|
340
|
+
if (!readSurvives) dark.push(table);
|
|
247
341
|
}
|
|
248
|
-
return
|
|
342
|
+
return dark.sort();
|
|
249
343
|
}
|
|
250
344
|
|
|
251
345
|
/** The plan's content address — recorded as `plan_ref` on the schema_log row. */
|
|
@@ -299,7 +393,9 @@ export function buildPlanSummary(plan: EdgePlan): string[] {
|
|
|
299
393
|
// rows, and a plan that drops a table's only read policy leaves that table returning nothing.
|
|
300
394
|
// 11 tables went dark in a real adoption plan that printed "0 notice(s)".
|
|
301
395
|
const { authzRemovals, unclassified } = partitionStatements(classifyGeneratedStatements(plan.statements).executable);
|
|
302
|
-
|
|
396
|
+
// Prefer what the mint resolved against live. Recomputing here would have only the statement
|
|
397
|
+
// text and would re-introduce the false positives the mint-time answer exists to avoid.
|
|
398
|
+
const dark = plan.dark ?? tablesLeftWithoutARead(plan.statements);
|
|
303
399
|
if (authzRemovals.length > 0) {
|
|
304
400
|
lines.push(`! ${authzRemovals.length} statement(s) REMOVE authorization (policies, grants, RLS). No data is lost; who can read it changes.`);
|
|
305
401
|
if (dark.length > 0) {
|
package/src/cli/exec-digest.ts
CHANGED
|
@@ -24,24 +24,63 @@
|
|
|
24
24
|
import { escapeLiteral } from './derived-apply.js';
|
|
25
25
|
|
|
26
26
|
/**
|
|
27
|
-
* Build the digest query
|
|
28
|
-
*
|
|
29
|
-
*
|
|
27
|
+
* Build the digest query. Returns SQL selecting a single `digest` column (md5 hex, or md5('')
|
|
28
|
+
* for an empty catalog).
|
|
29
|
+
*
|
|
30
|
+
* `schemas` IS ACCEPTED AND NO LONGER SCOPES THE DIGEST. It was the design error that made this
|
|
31
|
+
* guard fail in the field: a consumer ran 21 `CREATE OR REPLACE FUNCTION` in `auth` through
|
|
32
|
+
* db:exec and the guard stayed silent, because the deploy's schema set was `['public']` and the
|
|
33
|
+
* functions were simply outside the projection. **A guard that can be scoped can be scoped to
|
|
34
|
+
* blindness**, and the question db:exec asks is not "did the app's schemas move?" but "did the
|
|
35
|
+
* schema move AT ALL?". So the projection now covers every non-system schema, and no
|
|
36
|
+
* configuration — or missing configuration — can narrow it.
|
|
37
|
+
*
|
|
38
|
+
* The parameter stays in the signature deliberately: `@everystack/server` calls this through
|
|
39
|
+
* `@everystack/cli/exec`, so removing it would break every deployed ops Lambda that has not
|
|
40
|
+
* upgraded in lockstep. An old server passing `['public']` now gets full coverage.
|
|
41
|
+
*
|
|
42
|
+
* System schemas are excluded because they move for reasons that are not the caller's doing:
|
|
43
|
+
* `pg_toast` gains relations as tables acquire toastable columns, `pg_temp_*` appears and
|
|
44
|
+
* vanishes with sessions, and `pg_catalog`/`information_schema` are not the caller's to change.
|
|
30
45
|
*/
|
|
31
46
|
export function catalogDigestQuery(schemas: string[]): string {
|
|
32
47
|
if (schemas.length === 0) {
|
|
33
48
|
throw new Error('catalogDigestQuery needs at least one schema to digest — pass the app schemas.');
|
|
34
49
|
}
|
|
35
|
-
|
|
50
|
+
// Retained so the argument is not silently meaningless to a reader diffing this file: the
|
|
51
|
+
// value is validated exactly as before, then deliberately not used as a filter.
|
|
52
|
+
void schemas.map(escapeLiteral);
|
|
53
|
+
// A predicate, not an array: `= ANY(array_agg(nspname))` compares name to name[] and
|
|
54
|
+
// PostgreSQL has no such operator. Every branch below aliases pg_namespace as `n`.
|
|
55
|
+
const NS = `n.nspname NOT IN ('pg_catalog', 'information_schema', 'pg_toast')
|
|
56
|
+
AND n.nspname NOT LIKE 'pg_temp%' AND n.nspname NOT LIKE 'pg_toast_temp%'`;
|
|
36
57
|
return `
|
|
37
58
|
SELECT md5(coalesce(string_agg(line, E'\\n' ORDER BY line), '')) AS digest
|
|
38
59
|
FROM (
|
|
39
60
|
-- Relations: kind + persistence ONLY, never the volatile planner-stats columns (row-count,
|
|
40
61
|
-- page-count, freeze-horizon) — those move under INSERT/UPDATE/DELETE and autovacuum, and
|
|
41
62
|
-- would make the digest lie about pure DML.
|
|
42
|
-
|
|
63
|
+
-- relacl carries the GRANTs, and relrowsecurity/relforcerowsecurity the RLS posture. Both
|
|
64
|
+
-- were missing, so GRANT SELECT ON t TO anon, and ALTER TABLE t ENABLE ROW LEVEL SECURITY,
|
|
65
|
+
-- passed a gate advertising "rejects ANY schema change". A lane that cannot change the schema
|
|
66
|
+
-- but CAN change who may read it is not a DML-only lane. None of the three move under DML.
|
|
67
|
+
SELECT format('rel:%s.%s:%s:%s:%s:%s:%s', n.nspname, c.relname, c.relkind, c.relpersistence,
|
|
68
|
+
coalesce(c.relacl::text, ''), c.relrowsecurity, c.relforcerowsecurity) AS line
|
|
43
69
|
FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
44
|
-
WHERE
|
|
70
|
+
WHERE ${NS} AND c.relkind IN ('r', 'p', 'v', 'm', 'S', 'f')
|
|
71
|
+
UNION ALL
|
|
72
|
+
-- Row policies: a CREATE/ALTER/DROP POLICY rewrites who sees which rows and touches no other
|
|
73
|
+
-- catalog this projection reads, so without this it was invisible.
|
|
74
|
+
SELECT format('pol:%s.%s.%s:%s:%s:%s:%s:%s', n.nspname, c.relname, pol.polname, pol.polcmd,
|
|
75
|
+
pol.polpermissive,
|
|
76
|
+
coalesce((SELECT string_agg(r.rolname, ',' ORDER BY r.rolname)
|
|
77
|
+
FROM pg_roles r WHERE r.oid = ANY(pol.polroles)), 'PUBLIC'),
|
|
78
|
+
coalesce(pg_get_expr(pol.polqual, pol.polrelid), ''),
|
|
79
|
+
coalesce(pg_get_expr(pol.polwithcheck, pol.polrelid), ''))
|
|
80
|
+
FROM pg_policy pol
|
|
81
|
+
JOIN pg_class c ON c.oid = pol.polrelid
|
|
82
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
83
|
+
WHERE ${NS}
|
|
45
84
|
UNION ALL
|
|
46
85
|
-- Columns: number, type, typmod, not-null — the shape ALTER TABLE moves, DML never.
|
|
47
86
|
SELECT format('att:%s.%s.%s:%s:%s:%s:%s',
|
|
@@ -49,31 +88,34 @@ FROM (
|
|
|
49
88
|
FROM pg_attribute a
|
|
50
89
|
JOIN pg_class c ON c.oid = a.attrelid
|
|
51
90
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
52
|
-
WHERE
|
|
91
|
+
WHERE ${NS} AND a.attnum > 0 AND NOT a.attisdropped
|
|
53
92
|
UNION ALL
|
|
54
93
|
-- Constraints: the full textual definition (PK/FK/unique/check predicate).
|
|
55
94
|
SELECT format('con:%s.%s:%s', n.nspname, con.conname, pg_get_constraintdef(con.oid))
|
|
56
95
|
FROM pg_constraint con JOIN pg_namespace n ON n.oid = con.connamespace
|
|
57
|
-
WHERE
|
|
96
|
+
WHERE ${NS}
|
|
58
97
|
UNION ALL
|
|
59
98
|
-- Indexes: the full textual definition (columns, uniqueness, partial WHERE).
|
|
60
99
|
SELECT format('idx:%s.%s:%s', n.nspname, ic.relname, pg_get_indexdef(i.indexrelid))
|
|
61
100
|
FROM pg_index i
|
|
62
101
|
JOIN pg_class ic ON ic.oid = i.indexrelid
|
|
63
102
|
JOIN pg_namespace n ON n.oid = ic.relnamespace
|
|
64
|
-
WHERE
|
|
103
|
+
WHERE ${NS}
|
|
65
104
|
UNION ALL
|
|
66
|
-
-- Routines: name + arg types + a hash of the body
|
|
67
|
-
|
|
105
|
+
-- Routines: name + arg types + a hash of the body, PLUS proacl (the EXECUTE grants) and
|
|
106
|
+
-- prosecdef. A REVOKE that leaves a SECURITY DEFINER function PUBLIC-executable, or a GRANT
|
|
107
|
+
-- that opens one, is a privilege change and must trip the guard like any other.
|
|
108
|
+
SELECT format('proc:%s.%s:%s:%s:%s:%s', n.nspname, p.proname, p.proargtypes::text,
|
|
109
|
+
md5(coalesce(p.prosrc, '')), coalesce(p.proacl::text, ''), p.prosecdef)
|
|
68
110
|
FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace
|
|
69
|
-
WHERE
|
|
111
|
+
WHERE ${NS}
|
|
70
112
|
UNION ALL
|
|
71
113
|
-- Types: kind + ordered enum labels. CREATE TYPE / ALTER TYPE ADD VALUE move it.
|
|
72
114
|
SELECT format('type:%s.%s:%s:%s', n.nspname, t.typname, t.typtype,
|
|
73
115
|
coalesce((SELECT string_agg(e.enumlabel, ',' ORDER BY e.enumsortorder)
|
|
74
116
|
FROM pg_enum e WHERE e.enumtypid = t.oid), ''))
|
|
75
117
|
FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace
|
|
76
|
-
WHERE
|
|
118
|
+
WHERE ${NS}
|
|
77
119
|
) s
|
|
78
120
|
`.trim();
|
|
79
121
|
}
|
package/src/cli/git-descent.ts
CHANGED
|
@@ -32,7 +32,8 @@ import type { ModelDescriptor } from '@everystack/model';
|
|
|
32
32
|
import type { SchemaSnapshot } from './schema-introspect.js';
|
|
33
33
|
import type { AuthzContract } from './authz-contract.js';
|
|
34
34
|
import { fingerprintLive } from './schema-fingerprint.js';
|
|
35
|
-
import { predictLiveFingerprint } from './edge-plan.js';
|
|
35
|
+
import { predictLiveFingerprint, governedLiveFingerprint } from './edge-plan.js';
|
|
36
|
+
import { governedRolesForModels } from './schema-fingerprint.js';
|
|
36
37
|
|
|
37
38
|
/** Compile a models barrel on disk. The default rides the CLI's runtime (tsx). */
|
|
38
39
|
export type ModelsLoader = (barrel: string) => Promise<ModelDescriptor[]>;
|
|
@@ -174,7 +175,6 @@ export async function verifyDescent(
|
|
|
174
175
|
const barrel = path.basename(modelsAbs);
|
|
175
176
|
|
|
176
177
|
const candidates = enumerateModelTrees(modelsDir, repoRoot);
|
|
177
|
-
const liveFingerprint = fingerprintLive(live.snapshot, live.contract).hash;
|
|
178
178
|
|
|
179
179
|
const nodeModules = path.join(repoRoot, 'node_modules');
|
|
180
180
|
const materializeRoot = opts.materializeRoot
|
|
@@ -187,29 +187,36 @@ export async function verifyDescent(
|
|
|
187
187
|
try {
|
|
188
188
|
for (const candidate of candidates) {
|
|
189
189
|
const dest = path.join(materializeRoot, candidate.tree);
|
|
190
|
-
let
|
|
190
|
+
let declares: boolean;
|
|
191
191
|
// A swallowed error here skips the candidate — and if it was the DECLARING tree, the
|
|
192
192
|
// verdict silently falls through to 'drift'. Real compile failures are deterministic;
|
|
193
193
|
// a transient materialize/import failure (fd pressure, module-loader contention under
|
|
194
194
|
// a parallel test run or a busy CI box) is not — so a failed candidate gets exactly
|
|
195
195
|
// one retry, from a clean materialization, before it is recorded as a compile failure.
|
|
196
|
-
|
|
196
|
+
//
|
|
197
|
+
// The declares-test is GOVERNED on both sides, per candidate: the commit's
|
|
198
|
+
// models define the governed set, and the live hash is filtered through the
|
|
199
|
+
// SAME set as the prediction — comparing a governed prediction against the
|
|
200
|
+
// raw live hash can never match on a brownfield target with foreign roles.
|
|
201
|
+
const attempt = async (): Promise<boolean> => {
|
|
197
202
|
if (!fs.existsSync(dest)) materializeTree(candidate.tree, dest, repoRoot);
|
|
198
203
|
const models = await loader(path.join(dest, barrel));
|
|
199
|
-
|
|
204
|
+
const governedRoles = governedRolesForModels(models);
|
|
205
|
+
const predicted = predictLiveFingerprint(models, live.snapshot, live.contract, { schema: opts.schema, governedRoles });
|
|
206
|
+
return predicted === governedLiveFingerprint(live.snapshot, live.contract, governedRoles);
|
|
200
207
|
};
|
|
201
208
|
try {
|
|
202
209
|
try {
|
|
203
|
-
|
|
210
|
+
declares = await attempt();
|
|
204
211
|
} catch {
|
|
205
212
|
fs.rmSync(dest, { recursive: true, force: true });
|
|
206
|
-
|
|
213
|
+
declares = await attempt();
|
|
207
214
|
}
|
|
208
215
|
} catch (err: any) {
|
|
209
216
|
compileFailures.push(`${short(candidate.tree)} (at ${short(candidate.commits[0])}): ${err.message}`);
|
|
210
217
|
continue;
|
|
211
218
|
}
|
|
212
|
-
if (
|
|
219
|
+
if (!declares) continue;
|
|
213
220
|
|
|
214
221
|
for (const commit of candidate.commits) {
|
|
215
222
|
try {
|
|
@@ -255,7 +262,7 @@ export async function verifyDescent(
|
|
|
255
262
|
scannedTrees: candidates.length,
|
|
256
263
|
compileFailures,
|
|
257
264
|
reason:
|
|
258
|
-
`no committed models state declares the target's live fingerprint ${short(
|
|
265
|
+
`no committed models state declares the target's live fingerprint ${short(fingerprintLive(live.snapshot, live.contract).hash)} — ` +
|
|
259
266
|
`scanned ${candidates.length} historical tree(s) across all refs${failureNote}. ` +
|
|
260
267
|
'Either the database was hand-edited (drift) or the declaring history was rewritten. ' +
|
|
261
268
|
'Align the models with the live database first (db:pull, commit), ' +
|
package/src/cli/index.ts
CHANGED
|
@@ -363,11 +363,11 @@ Usage:
|
|
|
363
363
|
Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
|
|
364
364
|
everystack db:fingerprint [--stage <name> | --database-url <url>] [--models <barrel>] [--json] Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
|
|
365
365
|
everystack db:reconcile [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--only a,b] [--json] Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (atomic — DDL + provenance in one transaction) and records provenance + schema_log; --apply --stage runs credential-free in the ops Lambda (no admin URL on the deployer, the db:apply twin), --apply --database-url runs direct. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive. --only <schema.name,…> restricts the run to the named objects (surgical); with --rebuild it FORCES those to rebuild from source even when the hashes show no diff — the recovery exit when a mistaken --rebaseline left a self-consistent-but-wrong provenance row (the dependency cascade rebuilds their live dependents).
|
|
366
|
-
everystack db:refresh [--stage <name> | --database-url <url> | --direct] [--only a,b] [--verify-nonempty] [--list] Refresh the declared materialized views in dependency order, credential-free. --stage runs the whole refresh in the ops Lambda on the operator connection (no URL on the operator's machine — the data-lane twin of the reconcile lane); --database-url
|
|
366
|
+
everystack db:refresh [--stage <name> | --database-url <url> | --direct] [--only a,b] [--verify-nonempty] [--list] Refresh the declared materialized views in dependency order, credential-free. --stage runs the whole refresh in the ops Lambda on the operator connection (no URL on the operator's machine — the data-lane twin of the reconcile lane); --database-url refreshes over a local connection (dev); --stage <name> --direct resolves the stage's operator connection from its ops Lambda (credential never on argv) and refreshes CLI-side on an UNBOUNDED clock, for a matview set that exceeds the ops-Lambda 900s limit; --only refreshes a named subset (full identity or bare name); --verify-nonempty gates on populated-but-zero-rows matviews (the dark-panel check — a scoped EXISTS on just what was refreshed, so a promotion script gets the gate with no read authority) and FAILS naming any empty; --list previews the order without connecting. Plain REFRESH (ACCESS EXCLUSIVE); fail-fast, idempotent to re-run.
|
|
367
367
|
everystack db:sync [--database-url <url>] [--models <barrel>] [--schema-out <file.ts>] [--allow-drops] [--overwrite-drift] [--baseline] [--json] Make the database match your checkout — one verb, both layers: apply the state diff (tables+authz, one transaction, verified by re-diff), reconcile the derived layer against the declared descriptors, report the resulting fingerprint vs the models' declared one. Dev databases only (direct connection required); DROPs held back unless --allow-drops; derived drift refuses unless --overwrite-drift; exit 1 when not converged
|
|
368
368
|
everystack db:diff --from-models <barrel> [--to-models db/models/index.ts] [--allow-drops] [--check] [--json] The state edge between two declared states, NO database: the SQL db:generate would produce, computed purely — CI plan previews (--check exits 1 on a non-empty edge) and computed rollbacks (swap the flags)
|
|
369
|
-
everystack db:plan [--stage <name> | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -] Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only
|
|
370
|
-
everystack db:apply --plan <file.plan.json> [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--snapshot-ref <ref>] [--force-descent <snapshot-ref> --confirm] Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule, else refuse — "rebase first"), and for DESTRUCTIVE plans require --confirm always +
|
|
369
|
+
everystack db:plan [--stage <name> [--direct] | --database-url <url>] [--models <barrel>] [--allow-drops] [--out db.plan.json | --out -] Mint a verified edge against a target: asks the TARGET its fingerprint, diffs the models, writes ONE reviewable plan (edge + both endpoint fingerprints). Held drops refuse the mint (--allow-drops carries destruction explicitly). Read-only; plans are ephemeral, never committed. VENUES: --stage runs via the ops Lambda, which reads TWICE and refuses on disagreement — agreement there is a DETECTOR, not a verification. --stage --direct resolves the stage's operator connection from its IAM-gated ops Lambda, holds it in memory only, and reads ONCE over one session: the lane for a fingerprint you intend to trust, with the credential never on argv. --database-url is the local-dev venue (same read guarantee, but against a deployed stage it puts a privileged DSN on the command line)
|
|
370
|
+
everystack db:apply --plan <file.plan.json> [--database-url <url>] [--stage <name>] [--models <barrel>] [--confirm] [--snapshot-ref <ref>] [--force-descent <snapshot-ref> --confirm] Run a reviewed plan: verify the target is EXACTLY where the plan started (live fingerprint == plan.from, else refuse — the concurrency lock), verify the checkout DESCENDS from the commit declaring the target's state (the fast-forward rule, else refuse — "rebase first"), and for DESTRUCTIVE plans require --confirm always + an attested --snapshot-ref + the stage's approver set when declared (STS identity-verified). The STAGE lane (--stage without --direct) runs every catalog query in its own ops-Lambda invoke, so one read can be assembled from several containers: it reads the target TWICE and REFUSES when the two disagree (inconsistent containers), reports agreement as a NON-DETECTION (it cannot verify read consistency), and REFUSES a DESTRUCTIVE plan outright — destructive applies go over --database-url (direct) with the full ceremony. Every refusal that reaches the ops Lambda is recorded in schema_log. Apply as one transaction (plan_ref stamped), verify it landed exactly on plan.to; idempotent when already there
|
|
371
371
|
everystack db:check [--models <barrel>] [--schema-out <file.ts>] [--database-url <url>] [--json] The CI gate, per PR: the merged declared state must COMPOSE (models load, no duplicate tables, descriptors compile), every exposed RLS-enabled table must declare a read path (no force-RLS-with-no-read landmine that goes dark on the superuser drop), and generated artifacts must MATCH regeneration byte-for-byte; with a scratch PostgreSQL it builds the state from scratch on an ephemeral database (created + dropped) and requires fingerprint MATCH. Exit 1 on any failure; never touches a real target
|
|
372
372
|
everystack db:approvers --stage <name> [--set "cto,arn:..."] [--remove] Declare who can DESTROY: the stage's destructive-approver set (SSM parameter, admin-writable). Destructive db:apply runs are then identity-verified (STS) against it; --set '' disables destructive applies; --remove returns the stage to ceremony-only
|
|
373
373
|
everystack db:backfill [--database-url <url>] [--dir db/backfills] [--apply] [--mark-applied <file.sql>] [--json] One-shot data jobs in their own lane: plan shows applied (by CONTENT identity — renames/comment edits are no-ops) / pending (in order, unbounded-pass advisories) / blocked (a name that already ran in a different form — one-shot jobs are immutable). --apply runs each pending job as its own transaction, recorded in everystack.backfill_log (a failure rolls back alone, is recorded, stops the run); --mark-applied records without running. Never runs as a schema side effect; direct connection required
|