@everystack/cli 0.3.31 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -2
- package/src/cli/authz-compile.ts +5 -2
- package/src/cli/backfill.ts +1 -1
- package/src/cli/commands/db-branch.ts +28 -4
- package/src/cli/commands/db-check.ts +84 -33
- package/src/cli/commands/db-fingerprint.ts +7 -2
- package/src/cli/commands/db-generate.ts +71 -20
- package/src/cli/commands/db-pull.ts +37 -5
- package/src/cli/commands/db-reconcile.ts +132 -41
- package/src/cli/commands/db-sync.ts +54 -19
- package/src/cli/db-build.ts +9 -3
- package/src/cli/db-source.ts +5 -1
- package/src/cli/declared-derived.ts +136 -0
- package/src/cli/derived-apply.ts +40 -6
- package/src/cli/derived-compile.ts +327 -0
- package/src/cli/derived-grants.ts +96 -0
- package/src/cli/derived-introspect.ts +258 -21
- package/src/cli/derived-lint.ts +261 -0
- package/src/cli/derived-plan.ts +176 -10
- package/src/cli/derived-render.ts +320 -0
- package/src/cli/derived-source.ts +53 -125
- package/src/cli/git-descent.ts +15 -2
- package/src/cli/index.ts +6 -6
- package/src/cli/migration-compile.ts +38 -7
- package/src/cli/migration-generate.ts +43 -4
- package/src/cli/model-render.ts +106 -13
- package/src/cli/models-path.ts +1 -1
- package/src/cli/pipeline-path.ts +1 -1
- package/src/cli/schema-compile.ts +41 -10
- package/src/cli/schema-diff.ts +123 -17
- package/src/cli/schema-fingerprint.ts +28 -18
- package/src/cli/schema-introspect.ts +175 -8
- package/src/cli/schema-source.ts +75 -6
- package/src/cli/state-apply.ts +4 -1
package/src/cli/db-source.ts
CHANGED
|
@@ -83,7 +83,11 @@ export async function createUrlRunner(
|
|
|
83
83
|
);
|
|
84
84
|
}
|
|
85
85
|
const postgres = mod.default ?? mod;
|
|
86
|
-
|
|
86
|
+
// max_lifetime: null — the driver's default recycles a connection after a random 30–60
|
|
87
|
+
// minutes, resolving the in-flight query and THEN killing the session. Under reconcile's
|
|
88
|
+
// BEGIN-across-calls transaction that is a silent session swap mid-batch (the reconcile
|
|
89
|
+
// apply path also pins the backend pid and refuses bookkeeping if it moved).
|
|
90
|
+
const sql = postgres(url, { max: 1, max_lifetime: null, onnotice: () => {}, ...sslDefaults(url) });
|
|
87
91
|
return {
|
|
88
92
|
runner: async (query: string) => Array.from(await sql.unsafe(query)),
|
|
89
93
|
end: () => sql.end({ timeout: 5 }),
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The declared-DB loader — one call that turns the models barrel into everything the
|
|
3
|
+
* verbs consume beyond the models themselves: the descriptor-compiled derived layer
|
|
4
|
+
* (views/matviews/functions/triggers/escape-hatch objects, topologically ordered), the
|
|
5
|
+
* standalone sequences (state), and the pending table renames (so trigger provenance
|
|
6
|
+
* migrates instead of drop+create-ing). This is the B3 seam: the same loader feeds
|
|
7
|
+
* db:reconcile, db:sync, db:check, and db:generate — descriptors are live everywhere
|
|
8
|
+
* or nowhere.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import fs from 'node:fs/promises';
|
|
13
|
+
import { pathToFileURL } from 'node:url';
|
|
14
|
+
import type { Module, SequenceDescriptor, DerivedDescriptor } from '@everystack/model';
|
|
15
|
+
import { compileDerived } from './derived-compile.js';
|
|
16
|
+
import { compileTableRenames } from './schema-compile.js';
|
|
17
|
+
import type { SourceObject } from './derived-source.js';
|
|
18
|
+
import { resolveModelsPath } from './models-path.js';
|
|
19
|
+
|
|
20
|
+
export interface DeclaredDerived {
|
|
21
|
+
/** Descriptor-compiled derived objects, in dependency order. */
|
|
22
|
+
objects: SourceObject[];
|
|
23
|
+
/** The raw descriptors — the B2 gates (db:check) read abilities, not compiled SQL. */
|
|
24
|
+
derived: DerivedDescriptor[];
|
|
25
|
+
/** Standalone sequences (state layer — created before tables, fingerprinted). */
|
|
26
|
+
sequences: SequenceDescriptor[];
|
|
27
|
+
/** Pending table renames (`new qualified` → `old qualified`) from the models' markers. */
|
|
28
|
+
renamedTables: Record<string, string>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Barrel → normalized Module list (a bare `models` export wraps into one module). */
|
|
32
|
+
export async function loadModulesFrom(modelsPath: string): Promise<Module[]> {
|
|
33
|
+
const abs = path.resolve(modelsPath);
|
|
34
|
+
let mod: any;
|
|
35
|
+
try {
|
|
36
|
+
mod = await import(pathToFileURL(abs).href);
|
|
37
|
+
} catch (err: any) {
|
|
38
|
+
throw new Error(`Could not load modules from ${modelsPath}: ${err.message}`);
|
|
39
|
+
}
|
|
40
|
+
if (Array.isArray(mod.modules)) return mod.modules;
|
|
41
|
+
const models = mod.models ?? mod.default;
|
|
42
|
+
if (Array.isArray(models)) return [{ models, extensions: [], sequences: [], derived: [], functions: [] }];
|
|
43
|
+
throw new Error(`${modelsPath} must export a \`modules\` (or \`models\`) array.`);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* B7 — db/sql is RETIRED; descriptors are the single home. A project still carrying
|
|
48
|
+
* `.sql` files in db/sql gets this message from every verb that used to read the
|
|
49
|
+
* directory: a hard failure carrying the migration path, never a silent skip (silence
|
|
50
|
+
* would leave those objects unmanaged and, later, honestly-dropped). An empty leftover
|
|
51
|
+
* directory or non-.sql stragglers are harmless — only actual `.sql` files trip it.
|
|
52
|
+
*/
|
|
53
|
+
export async function retiredSqlDir(dir: string = 'db/sql'): Promise<string | null> {
|
|
54
|
+
let entries: string[];
|
|
55
|
+
try {
|
|
56
|
+
entries = await fs.readdir(path.resolve(dir));
|
|
57
|
+
} catch (err: any) {
|
|
58
|
+
// A missing directory has nothing to say. Anything else (EACCES, EIO) must NOT
|
|
59
|
+
// silently disarm the tombstone — an unreadable db/sql could still hold live sources.
|
|
60
|
+
if (err?.code === 'ENOENT' || err?.code === 'ENOTDIR') return null;
|
|
61
|
+
throw new Error(`could not inspect ${dir} for retired db/sql sources: ${err?.message ?? err}`);
|
|
62
|
+
}
|
|
63
|
+
const files = entries.filter((f) => f.endsWith('.sql')).sort();
|
|
64
|
+
if (files.length === 0) return null;
|
|
65
|
+
return (
|
|
66
|
+
`${dir} is retired — the derived layer is DECLARED as descriptors, and this directory is no longer read. ` +
|
|
67
|
+
`${files.length} file(s) still present: ${files.join(', ')}. ` +
|
|
68
|
+
`Migrate: declare each object with defineView / defineMaterializedView / defineFunction / defineSql on your module ` +
|
|
69
|
+
`(or run \`everystack db:pull\` against the live database to render them), delete this directory, commit, then run ` +
|
|
70
|
+
`\`db:reconcile --apply --rebaseline\` once — it verifies each live object is untouched since the last reconcile ` +
|
|
71
|
+
`and re-records the re-rendered source, no rebuild (objects the reconciler never touched want --baseline instead; ` +
|
|
72
|
+
`the flags compose). See docs/derived-objects.md.`
|
|
73
|
+
);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The detector, anchored to BOTH homes a checkout can carry: the CWD's `db/sql` (the
|
|
78
|
+
* classic layout) and the resolved models home's sibling `sql/` directory (monorepos
|
|
79
|
+
* running verbs from the repo root with `--models apps/x/db/models/index.ts` — the
|
|
80
|
+
* red-team's CWD-anchoring finding). First message wins; probes are deduped.
|
|
81
|
+
*/
|
|
82
|
+
export async function retiredSqlDirAnywhere(modelsFlag?: string): Promise<string | null> {
|
|
83
|
+
const probes = new Map<string, string>(); // resolved → display
|
|
84
|
+
const add = (dir: string): void => { probes.set(path.resolve(dir), dir); };
|
|
85
|
+
add('db/sql');
|
|
86
|
+
add(path.join(path.dirname(path.dirname(resolveModelsPath(modelsFlag))), 'sql'));
|
|
87
|
+
for (const dir of probes.values()) {
|
|
88
|
+
const message = await retiredSqlDir(dir);
|
|
89
|
+
if (message) return message;
|
|
90
|
+
}
|
|
91
|
+
return null;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* The `--sql-dir` refusal — never a dead end. The flag is retired, but the directory it
|
|
96
|
+
* points at may still hold live db/sql-era sources the default probes would never see;
|
|
97
|
+
* the refusal carries THAT directory's state and migration path, so dropping the flag
|
|
98
|
+
* doesn't strand the files (the red-team's dead-end-refusal finding).
|
|
99
|
+
*/
|
|
100
|
+
export async function retiredSqlDirFlagRefusal(sqlDirFlag: string): Promise<string> {
|
|
101
|
+
const head =
|
|
102
|
+
'--sql-dir is retired — the derived layer is DECLARED as descriptors ' +
|
|
103
|
+
'(defineView/defineMaterializedView/defineFunction/defineSql on your module) and no raw-SQL directory is read. Remove the flag.';
|
|
104
|
+
let theirs: string | null;
|
|
105
|
+
try {
|
|
106
|
+
theirs = await retiredSqlDir(sqlDirFlag);
|
|
107
|
+
} catch (err: any) {
|
|
108
|
+
return `${head}\n${err?.message ?? err}`;
|
|
109
|
+
}
|
|
110
|
+
return theirs
|
|
111
|
+
? `${head}\n${theirs}`
|
|
112
|
+
: `${head} (${sqlDirFlag} carries no .sql files — nothing left to migrate.)`;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Load the declared derived layer + sequences + renames from the models barrel.
|
|
117
|
+
* Returns null when no barrel exists (a models-less V1 project) — the verbs then
|
|
118
|
+
* run with no derived layer at all.
|
|
119
|
+
*/
|
|
120
|
+
export async function loadDeclaredDerived(modelsFlag?: string): Promise<DeclaredDerived | null> {
|
|
121
|
+
const modelsPath = resolveModelsPath(modelsFlag);
|
|
122
|
+
try {
|
|
123
|
+
await fs.access(path.resolve(modelsPath));
|
|
124
|
+
} catch {
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
const modules = await loadModulesFrom(modelsPath);
|
|
128
|
+
const models = modules.flatMap((m) => m.models);
|
|
129
|
+
const derived = modules.flatMap((m) => m.derived);
|
|
130
|
+
return {
|
|
131
|
+
objects: compileDerived(models, derived),
|
|
132
|
+
derived,
|
|
133
|
+
sequences: modules.flatMap((m) => m.sequences),
|
|
134
|
+
renamedTables: compileTableRenames(models, {}),
|
|
135
|
+
};
|
|
136
|
+
}
|
package/src/cli/derived-apply.ts
CHANGED
|
@@ -34,6 +34,12 @@ export const ENSURE_RECONCILER_SQL: string[] = [
|
|
|
34
34
|
def_hash text NOT NULL,
|
|
35
35
|
applied_at timestamptz NOT NULL DEFAULT now()
|
|
36
36
|
)`,
|
|
37
|
+
// Provenance records how to REMOVE what it recorded (the C1 closure): a 'sql'-kind
|
|
38
|
+
// object never joins the live catalog, so its recorded drop_sql is the only way a
|
|
39
|
+
// removal can drop it instead of silently orphaning it. Idempotent — legacy
|
|
40
|
+
// databases upgrade in place; their old rows (relations/functions) never need these.
|
|
41
|
+
`ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS kind text`,
|
|
42
|
+
`ALTER TABLE everystack.derived_provenance ADD COLUMN IF NOT EXISTS drop_sql text`,
|
|
37
43
|
`CREATE TABLE IF NOT EXISTS everystack.schema_log (
|
|
38
44
|
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
39
45
|
applied_at timestamptz NOT NULL DEFAULT now(),
|
|
@@ -56,6 +62,15 @@ export interface RenderedReconcile {
|
|
|
56
62
|
record: string[];
|
|
57
63
|
/** Identities whose provenance rows must be deleted (drops + prunes). */
|
|
58
64
|
remove: string[];
|
|
65
|
+
/** Provenance identity migrations (a table rename carried its triggers — same object, new identity). */
|
|
66
|
+
migrate: Array<{ from: string; to: string }>;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The structural trigger drop — `DROP TRIGGER … ON <table>`, never derivable from a
|
|
70
|
+
* plain identity (the red-team's C4). Shared by the plan (drop actions) and the
|
|
71
|
+
* command (recording drop_sql at provenance time). */
|
|
72
|
+
export function triggerDropSql(name: string, table: string): string {
|
|
73
|
+
return `DROP TRIGGER IF EXISTS "${name}" ON ${table}`;
|
|
59
74
|
}
|
|
60
75
|
|
|
61
76
|
const DROP_KEYWORD: Record<string, string> = {
|
|
@@ -96,7 +111,10 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
|
|
|
96
111
|
for (const action of plan.actions) {
|
|
97
112
|
switch (action.action) {
|
|
98
113
|
case 'drop': {
|
|
99
|
-
|
|
114
|
+
// The new kinds carry their drop on the action (triggers: composed from structure;
|
|
115
|
+
// 'sql' objects: the recorded/declared drop_sql) — the legacy kinds keep the
|
|
116
|
+
// keyword rendering they always had.
|
|
117
|
+
statements.push(action.dropSql ?? `DROP ${DROP_KEYWORD[action.kind]} IF EXISTS ${quoteQualified(action.identity)}`);
|
|
100
118
|
// A rebuild's drop is followed by its create; only a true removal loses provenance.
|
|
101
119
|
if (!plan.actions.some((a) => a.action === 'create' && a.identity === action.identity)) {
|
|
102
120
|
remove.push(action.identity);
|
|
@@ -125,6 +143,12 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
|
|
|
125
143
|
record.push(action.identity);
|
|
126
144
|
break;
|
|
127
145
|
}
|
|
146
|
+
// A re-render of an untouched live object: no DDL — the upsert re-records the new
|
|
147
|
+
// source hash against the live def hash the post-batch introspection reads.
|
|
148
|
+
case 'rebaseline': {
|
|
149
|
+
record.push(action.identity);
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
128
152
|
case 'prune': {
|
|
129
153
|
remove.push(action.identity);
|
|
130
154
|
break;
|
|
@@ -132,13 +156,23 @@ export function renderReconcileSql(plan: ReconcilePlan, source: SourceObject[]):
|
|
|
132
156
|
}
|
|
133
157
|
}
|
|
134
158
|
|
|
135
|
-
|
|
159
|
+
// Grant convergence rides the same batch, after every object exists. Idempotent
|
|
160
|
+
// REVOKE/GRANT — no provenance to record (ACLs are not in any hash).
|
|
161
|
+
for (const r of plan.regrants) statements.push(...r.statements);
|
|
162
|
+
|
|
163
|
+
return { statements, record, remove, migrate: [...plan.migrations] };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
export function renderProvenanceUpsert(identity: string, srcHash: string, defHash: string, kind?: string, dropSql?: string): string {
|
|
167
|
+
return `INSERT INTO everystack.derived_provenance (identity, src_hash, def_hash, kind, drop_sql, applied_at)
|
|
168
|
+
VALUES (${escapeLiteral(identity)}, ${escapeLiteral(srcHash)}, ${escapeLiteral(defHash)}, ${nullable(kind)}, ${nullable(dropSql)}, now())
|
|
169
|
+
ON CONFLICT (identity) DO UPDATE SET src_hash = EXCLUDED.src_hash, def_hash = EXCLUDED.def_hash, kind = EXCLUDED.kind, drop_sql = EXCLUDED.drop_sql, applied_at = now()`;
|
|
136
170
|
}
|
|
137
171
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
172
|
+
/** A table rename carried its triggers along — same object, new identity; the provenance
|
|
173
|
+
* row migrates instead of the object drop+create-ing. */
|
|
174
|
+
export function renderProvenanceMigrate(from: string, to: string): string {
|
|
175
|
+
return `UPDATE everystack.derived_provenance SET identity = ${escapeLiteral(to)} WHERE identity = ${escapeLiteral(from)}`;
|
|
142
176
|
}
|
|
143
177
|
|
|
144
178
|
export function renderProvenanceDelete(identity: string): string {
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The descriptor compiler — the declarative derived layer's front-end to the reconciler
|
|
3
|
+
* (Brick B1b of docs/plans/read-model-everywhere.md).
|
|
4
|
+
*
|
|
5
|
+
* defineView / defineMaterializedView / defineFunction / defineSql / trigger() render to
|
|
6
|
+
* the SAME `SourceObject` shape (and content hash) the retired db/sql parser produced
|
|
7
|
+
* (`hashSourceContent`) — so an object authored either way is ONE object, and the
|
|
8
|
+
* db/sql-era → descriptor migration reconciled as a no-op instead of a full rebuild. The
|
|
9
|
+
* reconciler core (content-hash identity, provenance, cascade, baseline) is reused
|
|
10
|
+
* untouched; the edges the NEW kinds need (trigger drops, explicit-drop objects) land
|
|
11
|
+
* with the plan/apply extensions (B1c).
|
|
12
|
+
*
|
|
13
|
+
* Ordering is a topological sort over DECLARED `dependsOn` refs (plus the deps that come
|
|
14
|
+
* free: a trigger's `execute:` function, its declaration-site relation; a function's
|
|
15
|
+
* `setof(ref)` return). Ties break by identity sort — never by declaration or import
|
|
16
|
+
* order, so the plan is stable under refactors. Model (table) refs are state-layer:
|
|
17
|
+
* always created before any compute, so they impose no edge here.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import type {
|
|
21
|
+
ModelDescriptor, DerivedDescriptor, ViewDescriptor, MaterializedViewDescriptor,
|
|
22
|
+
FunctionDescriptor, SqlDescriptor, TriggerSpec, Ability, SetofReturn, DependsOnRef,
|
|
23
|
+
} from '@everystack/model';
|
|
24
|
+
import { hashSourceContent, parseQualified, DECLARED_SOURCE_FILE, type Attachment, type SourceObject } from './derived-source.js';
|
|
25
|
+
import { findInvokerReachabilityGaps } from './derived-lint.js';
|
|
26
|
+
|
|
27
|
+
/** The provenance marker for descriptor-compiled objects (SourceObject.file). */
|
|
28
|
+
const DECLARED = DECLARED_SOURCE_FILE;
|
|
29
|
+
|
|
30
|
+
/** `public.x` renders bare (authored style — hash parity with hand-written sources); other schemas qualify. */
|
|
31
|
+
function renderName(schema: string, name: string): string {
|
|
32
|
+
return schema === 'public' ? name : `${schema}.${name}`;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** The declared name of a dependable ref, for rendering (`SETOF posts`) and identity. */
|
|
36
|
+
function refName(ref: SetofReturn['setof']): string {
|
|
37
|
+
if ('table' in ref) return renderName(ref.schema, ref.table);
|
|
38
|
+
return ref.name;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
// Grants — the same audience semantics as tables
|
|
43
|
+
// ---------------------------------------------------------------------------
|
|
44
|
+
|
|
45
|
+
/** Relation read grants: bare read → anon + authenticated (the table precedent);
|
|
46
|
+
* `{ role }` narrows; `{ columns }` scopes the SELECT to a column list. */
|
|
47
|
+
function relationGrants(target: string, abilities: readonly Ability[]): Attachment[] {
|
|
48
|
+
const out: Attachment[] = [];
|
|
49
|
+
for (const a of abilities) {
|
|
50
|
+
const roles = a.condition.role ? a.condition.role : 'anon, authenticated';
|
|
51
|
+
const cols = a.condition.columns?.length
|
|
52
|
+
? ` (${a.condition.columns.map((c) => `"${c}"`).join(', ')})`
|
|
53
|
+
: '';
|
|
54
|
+
out.push({ kind: 'grant', sql: `GRANT SELECT${cols} ON ${target} TO ${roles}` });
|
|
55
|
+
}
|
|
56
|
+
return out;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Function authz: `REVOKE ALL … FROM PUBLIC` UNCONDITIONALLY (PostgreSQL grants EXECUTE
|
|
60
|
+
* to PUBLIC by default — the silent-open inverse of the RLS trap), then the declared
|
|
61
|
+
* `can('execute', { role })` grants. */
|
|
62
|
+
function functionGrants(signature: string, abilities: readonly Ability[]): Attachment[] {
|
|
63
|
+
const out: Attachment[] = [{ kind: 'grant', sql: `REVOKE ALL ON FUNCTION ${signature} FROM PUBLIC` }];
|
|
64
|
+
for (const a of abilities) {
|
|
65
|
+
out.push({ kind: 'grant', sql: `GRANT EXECUTE ON FUNCTION ${signature} TO ${a.condition.role}` });
|
|
66
|
+
}
|
|
67
|
+
return out;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// ---------------------------------------------------------------------------
|
|
71
|
+
// Renderers — one CREATE (no trailing semicolon) + attachments per descriptor
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
|
|
74
|
+
function escapeLiteral(s: string): string {
|
|
75
|
+
return s.replace(/'/g, "''");
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function commentAttachment(kindWord: string, target: string, comment: string | undefined): Attachment[] {
|
|
79
|
+
return comment ? [{ kind: 'comment', sql: `COMMENT ON ${kindWord} ${target} IS '${escapeLiteral(comment)}'` }] : [];
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
function renderView(v: ViewDescriptor): { sql: string; attachments: Attachment[] } {
|
|
83
|
+
const { schema, name } = parseQualified(v.name);
|
|
84
|
+
const target = renderName(schema, name);
|
|
85
|
+
// security_invoker only when true — false is PostgreSQL's default, and emitting the
|
|
86
|
+
// default would break hash parity with hand-authored sources.
|
|
87
|
+
const invoker = v.securityInvoker ? ' WITH (security_invoker = true)' : '';
|
|
88
|
+
return {
|
|
89
|
+
sql: `CREATE VIEW ${target}${invoker} AS\n${v.as}`,
|
|
90
|
+
attachments: [
|
|
91
|
+
...commentAttachment('VIEW', target, v.comment),
|
|
92
|
+
...relationGrants(target, v.abilities),
|
|
93
|
+
],
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function renderMaterializedView(mv: MaterializedViewDescriptor): { sql: string; attachments: Attachment[] } {
|
|
98
|
+
const { schema, name } = parseQualified(mv.name);
|
|
99
|
+
const target = renderName(schema, name);
|
|
100
|
+
// populate: 'deferred' → WITH NO DATA; the existing rebuild machinery (derived-apply's
|
|
101
|
+
// objectSql) appends the REFRESH after creation — outside the AS-SELECT populate lock.
|
|
102
|
+
const noData = mv.populate === 'deferred' ? '\nWITH NO DATA' : '';
|
|
103
|
+
const indexes: Attachment[] = mv.indexes.map((ix) => {
|
|
104
|
+
// Same vocabulary and rendering rules as table indexes (Brick E): plain entries
|
|
105
|
+
// quote, raw entries (expressions/DESC/opclass) pass verbatim, names sanitize.
|
|
106
|
+
const nameParts = ix.columns.map((c) => c.replace(/\W+/g, '_').replace(/^_+|_+$/g, ''));
|
|
107
|
+
const idxName = `${name}_${nameParts.join('_')}_index`;
|
|
108
|
+
const cols = ix.columns
|
|
109
|
+
.map((c, i) => (ix.rawPositions?.[i] ? c : `"${c}"`))
|
|
110
|
+
.join(', ');
|
|
111
|
+
const unique = ix.isUnique ? 'UNIQUE ' : '';
|
|
112
|
+
const using = ix.method ? ` USING ${ix.method}` : '';
|
|
113
|
+
const include = ix.includeColumns?.length ? ` INCLUDE (${ix.includeColumns.map((c) => `"${c}"`).join(', ')})` : '';
|
|
114
|
+
const where = ix.predicate ? ` WHERE ${ix.predicate}` : '';
|
|
115
|
+
return { kind: 'index', sql: `CREATE ${unique}INDEX ${idxName} ON ${target}${using} (${cols})${include}${where}` };
|
|
116
|
+
});
|
|
117
|
+
return {
|
|
118
|
+
sql: `CREATE MATERIALIZED VIEW ${target} AS\n${mv.as}${noData}`,
|
|
119
|
+
attachments: [
|
|
120
|
+
...indexes,
|
|
121
|
+
...commentAttachment('MATERIALIZED VIEW', target, mv.comment),
|
|
122
|
+
...relationGrants(target, mv.abilities),
|
|
123
|
+
],
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** A dollar-quote tag the body cannot collide with. */
|
|
128
|
+
function dollarTag(body: string): string {
|
|
129
|
+
let tag = '$fn$';
|
|
130
|
+
for (let n = 1; body.includes(tag); n++) tag = `$fn${n}$`;
|
|
131
|
+
return tag;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** `name(text, integer)` — the signature GRANT/COMMENT statements reference. */
|
|
135
|
+
function functionSignature(target: string, fn: FunctionDescriptor): string {
|
|
136
|
+
return `${target}(${fn.args.map((a) => a.type).join(', ')})`;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
function renderFunction(fn: FunctionDescriptor): { sql: string; attachments: Attachment[] } {
|
|
140
|
+
const { schema, name } = parseQualified(fn.name);
|
|
141
|
+
const target = renderName(schema, name);
|
|
142
|
+
const args = fn.args
|
|
143
|
+
.map((a) => `${a.name} ${a.type}${a.default != null ? ` DEFAULT ${a.default}` : ''}`)
|
|
144
|
+
.join(', ');
|
|
145
|
+
const returns = typeof fn.returns === 'string' ? fn.returns : `SETOF ${refName(fn.returns.setof)}`;
|
|
146
|
+
const volatility = fn.volatility === 'volatile' ? '' : ` ${fn.volatility.toUpperCase()}`;
|
|
147
|
+
const security = fn.security === 'definer'
|
|
148
|
+
? ` SECURITY DEFINER SET search_path = ${(fn.searchPath ?? []).join(', ')}`
|
|
149
|
+
: '';
|
|
150
|
+
const tag = dollarTag(fn.body);
|
|
151
|
+
const signature = functionSignature(target, fn);
|
|
152
|
+
return {
|
|
153
|
+
sql:
|
|
154
|
+
`CREATE FUNCTION ${target}(${args})\n` +
|
|
155
|
+
`RETURNS ${returns}\n` +
|
|
156
|
+
`LANGUAGE ${fn.language}${volatility}${security}\n` +
|
|
157
|
+
`AS ${tag}\n${fn.body}\n${tag}`,
|
|
158
|
+
attachments: [
|
|
159
|
+
...commentAttachment('FUNCTION', signature, fn.comment),
|
|
160
|
+
...functionGrants(signature, fn.abilities),
|
|
161
|
+
],
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const TIMING_SQL = { before: 'BEFORE', after: 'AFTER', insteadOf: 'INSTEAD OF' } as const;
|
|
166
|
+
|
|
167
|
+
function renderTrigger(ownerSchema: string, ownerName: string, t: TriggerSpec): { sql: string; table: string } {
|
|
168
|
+
const table = renderName(ownerSchema, ownerName);
|
|
169
|
+
const events = t.events
|
|
170
|
+
.map((e) => (e === 'update' && t.of?.length ? `UPDATE OF ${t.of.map((c) => `"${c}"`).join(', ')}` : e.toUpperCase()))
|
|
171
|
+
.join(' OR ');
|
|
172
|
+
const when = t.condition ? ` WHEN (${t.condition})` : '';
|
|
173
|
+
const fnTarget = renderName(...(([q]) => [q.schema, q.name] as const)([parseQualified(t.execute.name)]));
|
|
174
|
+
return {
|
|
175
|
+
sql: `CREATE TRIGGER ${t.name} ${TIMING_SQL[t.timing]} ${events} ON ${table} FOR EACH ${t.forEach.toUpperCase()}${when} EXECUTE FUNCTION ${fnTarget}()`,
|
|
176
|
+
table,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
// ---------------------------------------------------------------------------
|
|
181
|
+
// The compiler — descriptors (+ model/view triggers) → topologically ordered SourceObjects
|
|
182
|
+
// ---------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
interface Node {
|
|
185
|
+
identity: string;
|
|
186
|
+
deps: string[];
|
|
187
|
+
build: (seq: number) => SourceObject;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function derivedIdentity(d: DerivedDescriptor): string {
|
|
191
|
+
const { schema, name } = parseQualified(d.name);
|
|
192
|
+
return `${schema}.${name}`;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function make(kind: SourceObject['kind'], rawName: string, sql: string, attachments: Attachment[], extra: Partial<SourceObject> = {}): (seq: number) => SourceObject {
|
|
196
|
+
const { schema, name } = parseQualified(rawName);
|
|
197
|
+
return (seq) => ({
|
|
198
|
+
kind, schema, name,
|
|
199
|
+
identity: extra.identity ?? `${schema}.${name}`,
|
|
200
|
+
sql, attachments,
|
|
201
|
+
hash: hashSourceContent(sql, attachments),
|
|
202
|
+
file: DECLARED, seq,
|
|
203
|
+
...extra,
|
|
204
|
+
});
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Compile the declared derived layer: every descriptor plus every trigger declared on the
|
|
209
|
+
* models and views, in dependency order. A trigger's `execute:` function must itself be in
|
|
210
|
+
* the derived set — a trigger cannot draw on a function nothing declares.
|
|
211
|
+
*/
|
|
212
|
+
export function compileDerived(models: readonly ModelDescriptor[], derived: readonly DerivedDescriptor[]): SourceObject[] {
|
|
213
|
+
// Invoker-view reachability is a COMPILE fail (B2): a granted-but-unreadable view is
|
|
214
|
+
// structurally broken — every verb that consumes descriptors refuses, not just db:check.
|
|
215
|
+
const unreachable = findInvokerReachabilityGaps(derived);
|
|
216
|
+
if (unreachable.length > 0) {
|
|
217
|
+
throw new Error(unreachable.map((g) => g.message).join('\n'));
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
const identities = new Set(derived.map(derivedIdentity));
|
|
221
|
+
const nodes: Node[] = [];
|
|
222
|
+
|
|
223
|
+
const depIdentities = (refs: readonly unknown[]): string[] =>
|
|
224
|
+
refs
|
|
225
|
+
.filter((r): r is DerivedDescriptor => typeof r === 'object' && r != null && !('table' in (r as object)))
|
|
226
|
+
.map((r) => derivedIdentity(r))
|
|
227
|
+
.filter((id) => identities.has(id));
|
|
228
|
+
|
|
229
|
+
// The DECLARED dependency set, verbatim: every ref (models included, in-set or not) as
|
|
230
|
+
// an identity — what B4's dependency drift verifies against the live catalog's edges.
|
|
231
|
+
// Distinct from the topo `deps` above, which keep only in-set derived identities.
|
|
232
|
+
const declaredIdentities = (refs: readonly DependsOnRef[]): string[] =>
|
|
233
|
+
[...new Set(refs.map((r) => ('table' in r ? `${r.schema}.${r.table}` : derivedIdentity(r))))].sort();
|
|
234
|
+
|
|
235
|
+
for (const d of derived) {
|
|
236
|
+
const identity = derivedIdentity(d);
|
|
237
|
+
switch (d.kind) {
|
|
238
|
+
case 'view': {
|
|
239
|
+
const { sql, attachments } = renderView(d);
|
|
240
|
+
nodes.push({ identity, deps: depIdentities(d.dependsOn), build: make('view', d.name, sql, attachments, { declaredDeps: declaredIdentities(d.dependsOn) }) });
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
case 'materialized view': {
|
|
244
|
+
const { sql, attachments } = renderMaterializedView(d);
|
|
245
|
+
nodes.push({ identity, deps: depIdentities(d.dependsOn), build: make('materialized view', d.name, sql, attachments, { declaredDeps: declaredIdentities(d.dependsOn) }) });
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
case 'function': {
|
|
249
|
+
const { sql, attachments } = renderFunction(d);
|
|
250
|
+
const setofDep = typeof d.returns === 'string' ? [] : depIdentities([d.returns.setof]);
|
|
251
|
+
nodes.push({ identity, deps: [...depIdentities(d.dependsOn), ...setofDep], build: make('function', d.name, sql, attachments) });
|
|
252
|
+
break;
|
|
253
|
+
}
|
|
254
|
+
case 'sql': {
|
|
255
|
+
// The drop is ALWAYS present on the compiled object — explicit from the descriptor,
|
|
256
|
+
// or derived for the simple (name-only DROP) kinds — so provenance always records
|
|
257
|
+
// how to remove it (removal never orphans; the C1 closure).
|
|
258
|
+
const q = parseQualified(d.name);
|
|
259
|
+
const drop = d.drop ?? `DROP ${d.objectKind.toUpperCase()} IF EXISTS ${renderName(q.schema, q.name)}`;
|
|
260
|
+
nodes.push({
|
|
261
|
+
identity,
|
|
262
|
+
deps: depIdentities(d.dependsOn),
|
|
263
|
+
build: make('sql', d.name, d.as, [], { objectKind: d.objectKind, drop }),
|
|
264
|
+
});
|
|
265
|
+
break;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Triggers declared on models and on views — the deps come free (execute fn; the
|
|
271
|
+
// declaration-site view when it is itself derived). The function MUST be declared.
|
|
272
|
+
const triggerOwners: Array<{ schema: string; name: string; ownerIdentity: string | null; triggers: TriggerSpec[] }> = [
|
|
273
|
+
...models.map((m) => ({ schema: m.schema, name: m.table, ownerIdentity: null, triggers: m.triggers })),
|
|
274
|
+
...derived.filter((d): d is ViewDescriptor => d.kind === 'view').map((v) => {
|
|
275
|
+
const q = parseQualified(v.name);
|
|
276
|
+
return { schema: q.schema, name: q.name, ownerIdentity: `${q.schema}.${q.name}`, triggers: v.triggers };
|
|
277
|
+
}),
|
|
278
|
+
];
|
|
279
|
+
for (const owner of triggerOwners) {
|
|
280
|
+
for (const t of owner.triggers) {
|
|
281
|
+
const fnIdentity = derivedIdentity(t.execute);
|
|
282
|
+
if (!identities.has(fnIdentity)) {
|
|
283
|
+
throw new Error(
|
|
284
|
+
`trigger '${t.name}' on ${owner.schema}.${owner.name}: its execute function '${t.execute.name}' is not in the derived set — declare it in defineModule({ derived: [...] }).`,
|
|
285
|
+
);
|
|
286
|
+
}
|
|
287
|
+
const { sql, table } = renderTrigger(owner.schema, owner.name, t);
|
|
288
|
+
const identity = `${owner.schema}.${owner.name}.${t.name}`;
|
|
289
|
+
nodes.push({
|
|
290
|
+
identity,
|
|
291
|
+
deps: [fnIdentity, ...(owner.ownerIdentity ? [owner.ownerIdentity] : [])],
|
|
292
|
+
build: make('trigger', `${owner.schema}.${t.name}`, sql, [], { identity, table }),
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
// Kahn's algorithm with a sorted ready-set: deterministic under any declaration order,
|
|
298
|
+
// ties broken by identity — never by import order. A cycle is a compile error.
|
|
299
|
+
const byIdentity = new Map(nodes.map((n) => [n.identity, n]));
|
|
300
|
+
const indegree = new Map(nodes.map((n) => [n.identity, 0]));
|
|
301
|
+
const dependents = new Map<string, string[]>();
|
|
302
|
+
for (const n of nodes) {
|
|
303
|
+
for (const dep of n.deps) {
|
|
304
|
+
indegree.set(n.identity, (indegree.get(n.identity) ?? 0) + 1);
|
|
305
|
+
dependents.set(dep, [...(dependents.get(dep) ?? []), n.identity]);
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
const ready = nodes.filter((n) => (indegree.get(n.identity) ?? 0) === 0).map((n) => n.identity).sort();
|
|
309
|
+
const ordered: SourceObject[] = [];
|
|
310
|
+
while (ready.length) {
|
|
311
|
+
const id = ready.shift()!;
|
|
312
|
+
ordered.push(byIdentity.get(id)!.build(ordered.length));
|
|
313
|
+
for (const dep of dependents.get(id) ?? []) {
|
|
314
|
+
const left = (indegree.get(dep) ?? 0) - 1;
|
|
315
|
+
indegree.set(dep, left);
|
|
316
|
+
if (left === 0) {
|
|
317
|
+
ready.push(dep);
|
|
318
|
+
ready.sort();
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
if (ordered.length !== nodes.length) {
|
|
323
|
+
const stuck = nodes.filter((n) => !ordered.some((o) => o.identity === n.identity)).map((n) => n.identity).sort();
|
|
324
|
+
throw new Error(`derived dependency cycle among: ${stuck.join(', ')} — a view cannot (transitively) depend on itself; PostgreSQL would refuse it too.`);
|
|
325
|
+
}
|
|
326
|
+
return ordered;
|
|
327
|
+
}
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* derived-grants — the grant half of B4's live drift
|
|
3
|
+
* (docs/plans/read-model-everywhere.md, "B4 — live drift + doctor").
|
|
4
|
+
*
|
|
5
|
+
* Content hashes see SOURCE changes; a hand-run GRANT/REVOKE on the live database touches
|
|
6
|
+
* no hash and used to be invisible. This module closes it with two pure pieces:
|
|
7
|
+
*
|
|
8
|
+
* - `parseGrantAttachments`: the declared grants, read back out of the object's own
|
|
9
|
+
* grant attachments. The grammar is OURS — the descriptor compiler emits
|
|
10
|
+
* every one of these statements — so the parse is narrow and total, never a general
|
|
11
|
+
* SQL parser. A `REVOKE ALL … FROM PUBLIC` declares PUBLIC as explicitly EMPTY, so a
|
|
12
|
+
* live default ACL (PostgreSQL grants functions EXECUTE to PUBLIC out of the box)
|
|
13
|
+
* diffs to the revoke.
|
|
14
|
+
* - `diffObjectGrants`: the reconcileGrants pattern from the table path — per grantee,
|
|
15
|
+
* revoke what live has that the source doesn't declare, grant what the source
|
|
16
|
+
* declares that live lacks. The declared source is authoritative for authz; grants
|
|
17
|
+
* converge without drift ceremony (rebuilds lose hand-edits, REVOKE/GRANT loses
|
|
18
|
+
* nothing but the drift itself).
|
|
19
|
+
*
|
|
20
|
+
* Column-scoped grants (`GRANT SELECT ("a", "b") …`) live in `pg_attribute.attacl`, which
|
|
21
|
+
* this drift pass does not introspect yet — an object carrying one is marked
|
|
22
|
+
* `hasColumnGrants` and its grant drift is skipped LOUDLY (the plan warns), never
|
|
23
|
+
* silently half-diffed.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
import type { Attachment } from './derived-source.js';
|
|
27
|
+
|
|
28
|
+
export interface ParsedGrants {
|
|
29
|
+
/** grantee → sorted privileges. PUBLIC present-with-[] means explicitly revoked. */
|
|
30
|
+
grants: Record<string, string[]>;
|
|
31
|
+
/** The `ON …` target as our statements spell it (`leaderboard`, `FUNCTION f(text)`). */
|
|
32
|
+
target: string | null;
|
|
33
|
+
/** A column-scoped grant is present — attacl is not introspected; skip drift, loudly. */
|
|
34
|
+
hasColumnGrants: boolean;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Our two emitters' shapes, nothing more:
|
|
38
|
+
// GRANT SELECT[ ("c1", "c2")] ON <target> TO r1[, r2…]
|
|
39
|
+
// GRANT EXECUTE ON FUNCTION <sig> TO r
|
|
40
|
+
// REVOKE ALL ON FUNCTION <sig> FROM PUBLIC
|
|
41
|
+
const GRANT_RE = /^GRANT\s+([A-Z]+)\s*(\([^)]*\))?\s+ON\s+(.+?)\s+TO\s+(.+)$/i;
|
|
42
|
+
const REVOKE_PUBLIC_RE = /^REVOKE\s+ALL\s+ON\s+(.+?)\s+FROM\s+PUBLIC$/i;
|
|
43
|
+
|
|
44
|
+
/** The declared grant contract, read back from an object's own grant attachments. */
|
|
45
|
+
export function parseGrantAttachments(attachments: readonly Attachment[]): ParsedGrants {
|
|
46
|
+
const grants: Record<string, Set<string>> = {};
|
|
47
|
+
let target: string | null = null;
|
|
48
|
+
let hasColumnGrants = false;
|
|
49
|
+
|
|
50
|
+
for (const a of attachments) {
|
|
51
|
+
if (a.kind !== 'grant') continue;
|
|
52
|
+
const revoke = a.sql.match(REVOKE_PUBLIC_RE);
|
|
53
|
+
if (revoke) {
|
|
54
|
+
target ??= revoke[1].trim();
|
|
55
|
+
grants.PUBLIC ??= new Set();
|
|
56
|
+
continue;
|
|
57
|
+
}
|
|
58
|
+
const grant = a.sql.match(GRANT_RE);
|
|
59
|
+
if (!grant) continue;
|
|
60
|
+
const [, privilege, columns, onTarget, roleList] = grant;
|
|
61
|
+
if (columns) {
|
|
62
|
+
hasColumnGrants = true;
|
|
63
|
+
continue; // attacl territory — declared, applied at create, not drift-checked yet
|
|
64
|
+
}
|
|
65
|
+
target ??= onTarget.trim();
|
|
66
|
+
for (const role of roleList.split(',').map((r) => r.trim()).filter(Boolean)) {
|
|
67
|
+
(grants[role] ??= new Set()).add(privilege.toUpperCase());
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const out: Record<string, string[]> = {};
|
|
72
|
+
for (const role of Object.keys(grants).sort()) out[role] = [...grants[role]].sort();
|
|
73
|
+
return { grants: out, target, hasColumnGrants };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The idempotent REVOKE/GRANT delta between the declared contract and the live ACLs —
|
|
78
|
+
* the table path's reconcileGrants, per derived object. Empty when they match.
|
|
79
|
+
*/
|
|
80
|
+
export function diffObjectGrants(
|
|
81
|
+
target: string,
|
|
82
|
+
declared: Record<string, string[]>,
|
|
83
|
+
live: Record<string, string[]>,
|
|
84
|
+
): string[] {
|
|
85
|
+
const out: string[] = [];
|
|
86
|
+
const grantees = new Set([...Object.keys(declared), ...Object.keys(live)]);
|
|
87
|
+
for (const grantee of [...grantees].sort()) {
|
|
88
|
+
const want = new Set(declared[grantee] ?? []);
|
|
89
|
+
const have = new Set(live[grantee] ?? []);
|
|
90
|
+
const toRevoke = [...have].filter((p) => !want.has(p)).sort();
|
|
91
|
+
const toGrant = [...want].filter((p) => !have.has(p)).sort();
|
|
92
|
+
if (toRevoke.length) out.push(`REVOKE ${toRevoke.join(', ')} ON ${target} FROM ${grantee}`);
|
|
93
|
+
if (toGrant.length) out.push(`GRANT ${toGrant.join(', ')} ON ${target} TO ${grantee}`);
|
|
94
|
+
}
|
|
95
|
+
return out;
|
|
96
|
+
}
|