@everystack/cli 0.4.12 → 0.4.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cli/commands/db-export.ts +147 -9
- package/src/cli/commands/db-pull.ts +22 -3
- package/src/cli/commands/db-swap.ts +5 -2
- package/src/cli/derived-render.ts +87 -2
- package/src/cli/index.ts +2 -2
- package/src/cli/model-render.ts +15 -2
- package/src/cli/schema-introspect.ts +25 -0
package/package.json
CHANGED
|
@@ -1,14 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* `everystack db:export --schema <name> --stage <n
|
|
3
|
-
* artifact (canonical-sync gap B). pg_dump ONE schema
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* `everystack db:export --schema <name> [--stage <n> | --database-url <url>]` — the schema-scoped,
|
|
3
|
+
* fingerprint-stamped artifact (canonical-sync gap B). pg_dump ONE schema, stamped with the
|
|
4
|
+
* DECLARED schema fingerprint from the checkout, so db:swap can gate an incoming artifact against
|
|
5
|
+
* the target's declared shape (declared-vs-declared: does the artifact and the stage agree on the
|
|
6
|
+
* schema's shape). everystack owns the export so that fingerprint is one formula both sides.
|
|
7
7
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
8
|
+
* Two venues, mirroring db:swap:
|
|
9
|
+
* - `--stage` (default): the dump runs in the ops Lambda (the DB is private) → the stage's S3
|
|
10
|
+
* artifact bucket, gzipped. Verification of the streaming path is a deploy smoke test.
|
|
11
|
+
* - `--database-url` (explicit flag only, never the env — the venue choice must be deliberate):
|
|
12
|
+
* the CLI runs pg_dump itself against a reachable database and writes a plain `-Fc` `.dump`
|
|
13
|
+
* plus the sibling `.meta.json` db:swap reads. This is the build-locally → publish → swap
|
|
14
|
+
* on-ramp: a consumer pipeline shells this instead of hand-rolling pg_dump and duplicating
|
|
15
|
+
* the fingerprint formula.
|
|
10
16
|
*/
|
|
11
17
|
|
|
18
|
+
import fs from 'node:fs';
|
|
19
|
+
import path from 'node:path';
|
|
20
|
+
import { randomBytes } from 'node:crypto';
|
|
21
|
+
import { spawn } from 'node:child_process';
|
|
12
22
|
import type { ModelDescriptor } from '@everystack/model';
|
|
13
23
|
import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
|
|
14
24
|
import { invokeAction } from '../aws.js';
|
|
@@ -17,11 +27,119 @@ import { resolveModelsPath } from '../models-path.js';
|
|
|
17
27
|
import { loadModels } from './db-generate.js';
|
|
18
28
|
import { loadDeclaredDerived } from '../declared-derived.js';
|
|
19
29
|
import { pgDumpPreflightError } from './db-backup.js';
|
|
30
|
+
import { pgEnvFromUrl } from './db.js';
|
|
31
|
+
import { utcStamp } from '../backup.js';
|
|
20
32
|
import { step, success, fail, info } from '../output.js';
|
|
21
33
|
|
|
22
34
|
const fmtBytes = (b?: number): string =>
|
|
23
35
|
b == null ? '?' : b >= 1024 ** 3 ? `${(b / 1024 ** 3).toFixed(1)} GB` : b >= 1024 ** 2 ? `${(b / 1024 ** 2).toFixed(1)} MB` : `${(b / 1024).toFixed(0)} KB`;
|
|
24
36
|
|
|
37
|
+
export type ExportVenue = { kind: 'local'; url: string } | { kind: 'stage'; stage: string | undefined };
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Which venue runs the dump. Local ONLY on an explicit `--database-url` — never the env, so a
|
|
41
|
+
* DATABASE_URL in the shell can't silently flip a stage export into a local one. Both flags at
|
|
42
|
+
* once is ambiguous and refused.
|
|
43
|
+
*/
|
|
44
|
+
export function resolveExportVenue(flags: Record<string, string>): ExportVenue {
|
|
45
|
+
const url = flags['database-url'];
|
|
46
|
+
if (url && flags.stage) {
|
|
47
|
+
throw new Error('db:export takes one venue: --database-url (dump a reachable database locally) OR --stage (dump the stage\'s private database in the ops Lambda) — not both.');
|
|
48
|
+
}
|
|
49
|
+
return url ? { kind: 'local', url } : { kind: 'stage', stage: flags.stage };
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Where a local artifact lands. Mirrors the S3 scheme (`<stamp>-<shortId>`, meta as a sibling)
|
|
54
|
+
* with the schema in the file name since there is no bucket prefix; the id says `local` where
|
|
55
|
+
* the S3 id says the stage. Plain `.dump` (no gzip): db:swap's pg_restore reads the file as-is.
|
|
56
|
+
*/
|
|
57
|
+
export function localArtifactPaths(
|
|
58
|
+
schema: string,
|
|
59
|
+
now: Date,
|
|
60
|
+
shortId: string,
|
|
61
|
+
opts: { out?: string; outDir?: string } = {},
|
|
62
|
+
): { dumpPath: string; metaPath: string; id: string } {
|
|
63
|
+
const stamp = utcStamp(now);
|
|
64
|
+
let dumpPath: string;
|
|
65
|
+
if (opts.out) {
|
|
66
|
+
if (!opts.out.endsWith('.dump')) {
|
|
67
|
+
throw new Error(`--out must end in .dump (got ${opts.out}) — db:swap resolves the sibling .meta.json from that suffix.`);
|
|
68
|
+
}
|
|
69
|
+
dumpPath = opts.out;
|
|
70
|
+
} else {
|
|
71
|
+
dumpPath = path.join(opts.outDir ?? '.', `${schema}-${stamp}-${shortId}.dump`);
|
|
72
|
+
}
|
|
73
|
+
return {
|
|
74
|
+
dumpPath,
|
|
75
|
+
metaPath: dumpPath.replace(/\.dump$/, '.meta.json'),
|
|
76
|
+
id: `${schema}/local/${stamp}-${shortId}`,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** The meta record the swap gate reads — same shape as the S3 sidecar, `path` instead of `key`. */
|
|
81
|
+
export function localArtifactMeta(opts: {
|
|
82
|
+
id: string;
|
|
83
|
+
schema: string;
|
|
84
|
+
createdAt: Date;
|
|
85
|
+
bytes: number;
|
|
86
|
+
path: string;
|
|
87
|
+
fingerprint: string;
|
|
88
|
+
}): Record<string, unknown> {
|
|
89
|
+
return {
|
|
90
|
+
id: opts.id,
|
|
91
|
+
schema: opts.schema,
|
|
92
|
+
stage: 'local',
|
|
93
|
+
createdAt: opts.createdAt.toISOString(),
|
|
94
|
+
bytes: opts.bytes,
|
|
95
|
+
path: opts.path,
|
|
96
|
+
fingerprint: opts.fingerprint,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** pg_dump args for the local venue — same dump shape as the ops venue, written straight to a file. */
|
|
101
|
+
export function pgDumpLocalArgs(schema: string, dumpPath: string): string[] {
|
|
102
|
+
return ['-Fc', '--no-owner', '--no-privileges', '--no-comments', `--schema=${schema}`, '-f', dumpPath];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Run the local export: pg_dump one schema to a `-Fc` file, then write the meta sidecar. The
|
|
107
|
+
* connection rides PG* env (never argv). A failed dump removes the partial file — a truncated
|
|
108
|
+
* artifact never ships. Throws on failure; the command wraps it.
|
|
109
|
+
*/
|
|
110
|
+
export async function runLocalExport(opts: {
|
|
111
|
+
url: string;
|
|
112
|
+
schema: string;
|
|
113
|
+
fingerprint: string;
|
|
114
|
+
out?: string;
|
|
115
|
+
outDir?: string;
|
|
116
|
+
}): Promise<{ id: string; dumpPath: string; metaPath: string; bytes: number }> {
|
|
117
|
+
const now = new Date();
|
|
118
|
+
const shortId = randomBytes(3).toString('hex');
|
|
119
|
+
const { dumpPath, metaPath, id } = localArtifactPaths(opts.schema, now, shortId, opts);
|
|
120
|
+
|
|
121
|
+
const env = { ...process.env, ...pgEnvFromUrl(opts.url) };
|
|
122
|
+
const child = spawn('pg_dump', pgDumpLocalArgs(opts.schema, dumpPath), { env, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
123
|
+
let stderr = '';
|
|
124
|
+
child.stderr.on('data', (d) => { stderr += d.toString(); });
|
|
125
|
+
try {
|
|
126
|
+
await new Promise<void>((resolve, reject) => {
|
|
127
|
+
child.on('error', reject);
|
|
128
|
+
child.on('close', (code) =>
|
|
129
|
+
code === 0 ? resolve() : reject(new Error(`pg_dump exited ${code}: ${stderr.trim()}`)));
|
|
130
|
+
});
|
|
131
|
+
} catch (err) {
|
|
132
|
+
fs.rmSync(dumpPath, { force: true });
|
|
133
|
+
throw err;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const bytes = fs.statSync(dumpPath).size;
|
|
137
|
+
fs.writeFileSync(metaPath, JSON.stringify(localArtifactMeta({
|
|
138
|
+
id, schema: opts.schema, createdAt: now, bytes, path: dumpPath, fingerprint: opts.fingerprint,
|
|
139
|
+
}), null, 2));
|
|
140
|
+
return { id, dumpPath, metaPath, bytes };
|
|
141
|
+
}
|
|
142
|
+
|
|
25
143
|
export async function dbExportCommand(flags: Record<string, string>): Promise<void> {
|
|
26
144
|
const schema = flags.schema;
|
|
27
145
|
if (!schema) {
|
|
@@ -29,6 +147,14 @@ export async function dbExportCommand(flags: Record<string, string>): Promise<vo
|
|
|
29
147
|
process.exit(1);
|
|
30
148
|
}
|
|
31
149
|
|
|
150
|
+
let venue: ExportVenue;
|
|
151
|
+
try {
|
|
152
|
+
venue = resolveExportVenue(flags);
|
|
153
|
+
} catch (err: any) {
|
|
154
|
+
fail(err.message);
|
|
155
|
+
process.exit(1);
|
|
156
|
+
}
|
|
157
|
+
|
|
32
158
|
// The DECLARED fingerprint of just this schema — the stamp the swap gate compares against.
|
|
33
159
|
const modelsPath = resolveModelsPath(flags.models);
|
|
34
160
|
let fingerprint: string;
|
|
@@ -43,10 +169,22 @@ export async function dbExportCommand(flags: Record<string, string>): Promise<vo
|
|
|
43
169
|
process.exit(1);
|
|
44
170
|
}
|
|
45
171
|
|
|
172
|
+
if (venue.kind === 'local') {
|
|
173
|
+
step(`Running pg_dump --schema=${schema} against --database-url...`);
|
|
174
|
+
try {
|
|
175
|
+
const res = await runLocalExport({ url: venue.url, schema, fingerprint, out: flags.out });
|
|
176
|
+
success(`Artifact ${res.dumpPath} (${fmtBytes(res.bytes)}, fingerprint ${fingerprint.slice(0, 12)}, meta ${res.metaPath}). Deploy with: everystack db:swap --schema ${schema} --from ${res.dumpPath} --database-url <target>`);
|
|
177
|
+
} catch (err: any) {
|
|
178
|
+
fail(`Export failed: ${err.message}`);
|
|
179
|
+
process.exit(1);
|
|
180
|
+
}
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
183
|
+
|
|
46
184
|
step('Resolving deployed config...');
|
|
47
185
|
let config: CliConfig;
|
|
48
186
|
try {
|
|
49
|
-
config = await resolveConfig(
|
|
187
|
+
config = await resolveConfig(venue.stage);
|
|
50
188
|
} catch (err: any) {
|
|
51
189
|
fail(err.message);
|
|
52
190
|
process.exit(1);
|
|
@@ -68,7 +206,7 @@ export async function dbExportCommand(flags: Record<string, string>): Promise<vo
|
|
|
68
206
|
step(`Running pg_dump --schema=${schema} → S3 (this may take a while for large schemas)...`);
|
|
69
207
|
let result: any;
|
|
70
208
|
try {
|
|
71
|
-
result = await invokeAction(config.region, opsFunction(config), 'db:export', { schema, stage:
|
|
209
|
+
result = await invokeAction(config.region, opsFunction(config), 'db:export', { schema, stage: venue.stage, fingerprint });
|
|
72
210
|
} catch (err: any) {
|
|
73
211
|
fail(`Export failed: ${err.message}`);
|
|
74
212
|
process.exit(1);
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
|
|
30
30
|
import fs from 'node:fs/promises';
|
|
31
31
|
import path from 'node:path';
|
|
32
|
-
import { introspectSchema } from '../schema-introspect.js';
|
|
32
|
+
import { introspectSchema, MATVIEW_COLUMNS_SQL, matviewColumnsByIdentity, type ColumnRow, type ColumnSchema } from '../schema-introspect.js';
|
|
33
33
|
import { introspectDerived, type DerivedCatalog } from '../derived-introspect.js';
|
|
34
34
|
import { renderDerivedSource, renderDerivedFile, type DerivedRenderResult } from '../derived-render.js';
|
|
35
35
|
import { renderModelSource, renderModelFiles, pullableTables, modelVarName, ABILITY_PRESETS } from '../model-render.js';
|
|
@@ -84,8 +84,11 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
|
|
|
84
84
|
process.exit(1);
|
|
85
85
|
}
|
|
86
86
|
|
|
87
|
+
const matviewsAsTables = flags['matviews-as-tables'] === 'true';
|
|
88
|
+
|
|
87
89
|
let current;
|
|
88
90
|
let derivedCatalog: DerivedCatalog | undefined;
|
|
91
|
+
let matviewColumns: Map<string, ColumnSchema[]> | undefined;
|
|
89
92
|
try {
|
|
90
93
|
let runner: QueryRunner;
|
|
91
94
|
let end: (() => Promise<void>) | undefined;
|
|
@@ -103,6 +106,11 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
|
|
|
103
106
|
// The derived layer rides the same pull (B5) — views/matviews/functions/sequences
|
|
104
107
|
// render as descriptors; adoption is pull → commit → --baseline → clean reconcile.
|
|
105
108
|
derivedCatalog = await introspectDerived(runner);
|
|
109
|
+
// --matviews-as-tables: the flip needs real fields — one extra catalog read for the
|
|
110
|
+
// matview columns the derived layer (definition-only) doesn't carry.
|
|
111
|
+
if (matviewsAsTables) {
|
|
112
|
+
matviewColumns = matviewColumnsByIdentity((await runner(MATVIEW_COLUMNS_SQL)) as ColumnRow[]);
|
|
113
|
+
}
|
|
106
114
|
await end?.();
|
|
107
115
|
} catch (err: any) {
|
|
108
116
|
fail(err.message);
|
|
@@ -131,12 +139,22 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
|
|
|
131
139
|
const derived: DerivedRenderResult = renderDerivedSource(derivedCatalog ?? { objects: [], edges: [], provenance: [] }, {
|
|
132
140
|
knownTables,
|
|
133
141
|
sequences: current.sequences,
|
|
142
|
+
...(matviewsAsTables ? {
|
|
143
|
+
matviewsAsTables: {
|
|
144
|
+
columns: matviewColumns ?? new Map(),
|
|
145
|
+
enums: new Map((current.enums ?? []).map((e) => [e.name, e.values])),
|
|
146
|
+
},
|
|
147
|
+
} : {}),
|
|
134
148
|
});
|
|
135
149
|
for (const w of derived.warnings) caution(w);
|
|
136
150
|
const derivedCount = derived.names.length + derived.sequenceNames.length;
|
|
137
151
|
if (derivedCount > 0) {
|
|
138
152
|
detail(`Derived layer: ${derived.names.length} descriptor(s) + ${derived.sequenceNames.length} sequence(s) rendered (adopt with db:reconcile --baseline after committing).`);
|
|
139
153
|
}
|
|
154
|
+
if (derived.materializedTableNames.length) {
|
|
155
|
+
detail(`${derived.materializedTableNames.length} matview(s) rendered as defineMaterializedTable — these are MODELS: wire them into your module's models array (spread \`materializedTables\` from the derived file).`);
|
|
156
|
+
caution('Matviews carry no PK/NOT NULL — the rendered fields are nullable and unkeyed; tighten (.primaryKey()/.notNull()) as you review. Unique matview indexes render as index().unique() constraints.');
|
|
157
|
+
}
|
|
140
158
|
|
|
141
159
|
// The standalone derived file. The models output (below) then omits the derived
|
|
142
160
|
// layer — it lives here, and the module wires both: defineModule({ models, sequences, derived }).
|
|
@@ -149,8 +167,9 @@ export async function dbPullCommand(flags: Record<string, string>): Promise<void
|
|
|
149
167
|
fail(`Could not write ${derivedOut}: ${err.message}`);
|
|
150
168
|
process.exit(1);
|
|
151
169
|
}
|
|
152
|
-
|
|
153
|
-
|
|
170
|
+
const mtPart = derived.materializedTableNames.length ? ` + ${derived.materializedTableNames.length} materialized table(s)` : '';
|
|
171
|
+
ok(`Wrote ${path.relative(process.cwd(), outPath)} — ${derived.names.length} descriptor(s) + ${derived.sequenceNames.length} sequence(s)${mtPart}, self-contained.`);
|
|
172
|
+
note(`Wire it on your module — defineModule({ models${derived.materializedTableNames.length ? ': [...models, ...materializedTables]' : ''}${derived.sequenceNames.length ? ', sequences' : ''}, derived }) — then adopt with db:reconcile --apply --baseline (--rebaseline over db/sql-era provenance).`);
|
|
154
173
|
if (!flags.out) {
|
|
155
174
|
// The brownfield case: the barrel is hand-maintained; leave the models alone.
|
|
156
175
|
note('Models were NOT rendered (--derived-out extracts the derived layer only; add --out to render models too).');
|
|
@@ -86,8 +86,11 @@ async function restoreIntoIncoming(url: string, artifactPath: string, schema: st
|
|
|
86
86
|
]);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
|
-
/**
|
|
90
|
-
|
|
89
|
+
/**
|
|
90
|
+
* Load the artifact's meta (fingerprint) from a sibling `.meta.json`, or fall back to a flag.
|
|
91
|
+
* Exported so the db:export local venue can prove its meta round-trips into this gate.
|
|
92
|
+
*/
|
|
93
|
+
export function readArtifactFingerprint(artifactPath: string, flag?: string): string | null {
|
|
91
94
|
if (flag) return flag;
|
|
92
95
|
const metaPath = artifactPath.replace(/\.dump(\.gz)?$/, '.meta.json');
|
|
93
96
|
try {
|
|
@@ -14,9 +14,9 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import type { DerivedCatalog, LiveObject } from './derived-introspect.js';
|
|
17
|
-
import type { SequenceSchema } from './schema-introspect.js';
|
|
17
|
+
import type { ColumnSchema, SequenceSchema, TableSchema } from './schema-introspect.js';
|
|
18
18
|
import { parseIndexDefinition } from './schema-introspect.js';
|
|
19
|
-
import { modelFileName } from './model-render.js';
|
|
19
|
+
import { modelFileName, renderFieldLines } from './model-render.js';
|
|
20
20
|
|
|
21
21
|
export interface DerivedRenderResult {
|
|
22
22
|
/** The source block: `export const … = defineView(…)` etc., dependency-ordered. */
|
|
@@ -25,6 +25,9 @@ export interface DerivedRenderResult {
|
|
|
25
25
|
names: string[];
|
|
26
26
|
/** Rendered sequence variable names — feed `defineModule({ sequences })`. */
|
|
27
27
|
sequenceNames: string[];
|
|
28
|
+
/** defineMaterializedTable variable names (--matviews-as-tables) — these are MODELS:
|
|
29
|
+
* feed `defineModule({ models })`, never the derived array. */
|
|
30
|
+
materializedTableNames: string[];
|
|
28
31
|
/** Model-package symbols the block uses (defineView, can, sql, arg, index, …). */
|
|
29
32
|
imports: string[];
|
|
30
33
|
/** Qualified tables the block references through knownTables — what a standalone
|
|
@@ -39,6 +42,17 @@ export interface DerivedRenderOptions {
|
|
|
39
42
|
knownTables: Map<string, string>;
|
|
40
43
|
/** Standalone sequences from the snapshot (serial-owned stay implicit). */
|
|
41
44
|
sequences?: SequenceSchema[];
|
|
45
|
+
/**
|
|
46
|
+
* `db:pull --matviews-as-tables` (the gap-B consumer flip): render every matview as
|
|
47
|
+
* defineMaterializedTable with fields from its LIVE columns (matview identity → columns),
|
|
48
|
+
* instead of a defineMaterializedView descriptor. Matviews carry no PK/NOT NULL, so the
|
|
49
|
+
* fields land nullable and unkeyed — honest to the catalog; the consumer tightens on adoption.
|
|
50
|
+
*/
|
|
51
|
+
matviewsAsTables?: {
|
|
52
|
+
columns: Map<string, ColumnSchema[]>;
|
|
53
|
+
/** Enum name → values, for enum-typed matview columns (same map the model render uses). */
|
|
54
|
+
enums?: Map<string, string[]>;
|
|
55
|
+
};
|
|
42
56
|
}
|
|
43
57
|
|
|
44
58
|
const PLAIN_IDENT = /^[a-z_][a-z0-9_$]*$/;
|
|
@@ -130,10 +144,28 @@ function renderIndexBuilder(indexdef: string): string | null {
|
|
|
130
144
|
return s;
|
|
131
145
|
}
|
|
132
146
|
|
|
147
|
+
/**
|
|
148
|
+
* One matview index → the builder for a MODEL's `constraints: [...]` — same Brick E parse as
|
|
149
|
+
* {@link renderIndexBuilder}, but plain identifiers camelize to FIELD KEYS (the model
|
|
150
|
+
* constraints idiom; the compiler snake_cases them back), while expressions stay raw sql``.
|
|
151
|
+
*/
|
|
152
|
+
function renderTableIndexBuilder(indexdef: string): string | null {
|
|
153
|
+
const parsed = parseIndexDefinition(indexdef);
|
|
154
|
+
if (!parsed) return null;
|
|
155
|
+
const entry = (c: string): string => (PLAIN_IDENT.test(c) ? tsString(toCamelCase(c)) : `sql\`${tsTemplate(c)}\``);
|
|
156
|
+
let s = `index(${parsed.columns.map(entry).join(', ')})`;
|
|
157
|
+
if (parsed.using) s += `.using(${tsString(parsed.using)})`;
|
|
158
|
+
if (parsed.unique) s += '.unique()';
|
|
159
|
+
if (parsed.include?.length) s += `.include(${parsed.include.map((c) => tsString(toCamelCase(c))).join(', ')})`;
|
|
160
|
+
if (parsed.where) s += `.where(sql\`${tsTemplate(parsed.where)}\`)`;
|
|
161
|
+
return s;
|
|
162
|
+
}
|
|
163
|
+
|
|
133
164
|
export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRenderOptions): DerivedRenderResult {
|
|
134
165
|
const warnings: string[] = [];
|
|
135
166
|
const lines: string[] = [];
|
|
136
167
|
const names: string[] = [];
|
|
168
|
+
const materializedTableNames: string[] = [];
|
|
137
169
|
const used = new Set<string>();
|
|
138
170
|
const need = (sym: string): void => { used.add(sym); };
|
|
139
171
|
|
|
@@ -227,6 +259,53 @@ export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRender
|
|
|
227
259
|
lines.push(`// FIXME: ${o.identity} skipped — live grants (${grantText}) are not expressible as abilities (views are read surfaces).`, '');
|
|
228
260
|
continue;
|
|
229
261
|
}
|
|
262
|
+
|
|
263
|
+
// --matviews-as-tables (the gap-B consumer flip): the matview renders as a
|
|
264
|
+
// defineMaterializedTable MODEL — introspected fields, index constraints on field
|
|
265
|
+
// keys, the defining query as metadata. Same topo slot (its dependents still
|
|
266
|
+
// reference it by variable); the name lands in materializedTableNames, not names.
|
|
267
|
+
if (o.kind === 'materialized view' && opts.matviewsAsTables) {
|
|
268
|
+
const cols = opts.matviewsAsTables.columns.get(o.identity);
|
|
269
|
+
if (!cols || cols.length === 0) {
|
|
270
|
+
warnings.push(`${o.identity}: no live columns found for the table render — rendered as defineMaterializedView; re-pull or flip it by hand.`);
|
|
271
|
+
} else {
|
|
272
|
+
if (o.populated === false) {
|
|
273
|
+
warnings.push(`${o.identity}: the live matview is UNPOPULATED — rendered anyway (a materialized table's contents are pipeline-owned).`);
|
|
274
|
+
}
|
|
275
|
+
const shape: TableSchema = {
|
|
276
|
+
table: o.identity, columns: cols,
|
|
277
|
+
primaryKey: [], uniques: [], checks: [], foreignKeys: [], indexes: [],
|
|
278
|
+
};
|
|
279
|
+
const fieldLines = renderFieldLines(shape, new Set(), opts.matviewsAsTables.enums ?? new Map());
|
|
280
|
+
const builders = o.indexes.map((def) => {
|
|
281
|
+
const b = renderTableIndexBuilder(def);
|
|
282
|
+
if (b === null) warnings.push(`${o.identity}: index not renderable: ${def}`);
|
|
283
|
+
return b;
|
|
284
|
+
}).filter((b): b is string => b !== null);
|
|
285
|
+
if (builders.length) need('index');
|
|
286
|
+
if (abilities.length) need('can');
|
|
287
|
+
need('defineMaterializedTable');
|
|
288
|
+
need('field');
|
|
289
|
+
need('sql');
|
|
290
|
+
lines.push(
|
|
291
|
+
...fixmes,
|
|
292
|
+
`export const ${varName} = defineMaterializedTable(${tsString(declaredName(o))}, {`,
|
|
293
|
+
` ${abilities.length ? `abilities: [${abilities.join(', ')}],` : 'private: true,'}`,
|
|
294
|
+
' fields: {',
|
|
295
|
+
...fieldLines,
|
|
296
|
+
' },',
|
|
297
|
+
...(builders.length ? [` constraints: [${builders.join(', ')}],`] : []),
|
|
298
|
+
...(refs.length ? [` dependsOn: [${refs.join(', ')}],`] : []),
|
|
299
|
+
...(o.comment ? [` comment: ${tsString(o.comment)},`] : []),
|
|
300
|
+
` as: sql\`${tsTemplate(o.definition.trim().replace(/;$/, ''))}\`,`,
|
|
301
|
+
'});',
|
|
302
|
+
'',
|
|
303
|
+
);
|
|
304
|
+
materializedTableNames.push(varName);
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
230
309
|
const props: string[] = [];
|
|
231
310
|
if (o.kind === 'view') props.push(`securityInvoker: ${o.securityInvoker === true},`);
|
|
232
311
|
if (abilities.length === 0) props.push('private: true,');
|
|
@@ -327,6 +406,7 @@ export function renderDerivedSource(catalog: DerivedCatalog, opts: DerivedRender
|
|
|
327
406
|
block: lines.join('\n').trimEnd(),
|
|
328
407
|
names,
|
|
329
408
|
sequenceNames,
|
|
409
|
+
materializedTableNames,
|
|
330
410
|
imports: [...used].sort(),
|
|
331
411
|
modelRefs: [...modelRefs].sort(),
|
|
332
412
|
warnings,
|
|
@@ -359,6 +439,11 @@ export function renderDerivedFile(result: DerivedRenderResult, knownTables: Map<
|
|
|
359
439
|
if (result.sequenceNames.length) {
|
|
360
440
|
lines.push(`export const sequences = [\n${result.sequenceNames.map((n) => ` ${n},`).join('\n')}\n];`);
|
|
361
441
|
}
|
|
442
|
+
// Materialized tables (--matviews-as-tables) are MODELS: the consumer spreads this array
|
|
443
|
+
// into their module's models — defineModule({ models: [...models, ...materializedTables] }).
|
|
444
|
+
if (result.materializedTableNames.length) {
|
|
445
|
+
lines.push(`export const materializedTables = [\n${result.materializedTableNames.map((n) => ` ${n},`).join('\n')}\n];`);
|
|
446
|
+
}
|
|
362
447
|
if (result.names.length) {
|
|
363
448
|
lines.push(`export const derived = [\n${result.names.map((n) => ` ${n},`).join('\n')}\n];`);
|
|
364
449
|
}
|
package/src/cli/index.ts
CHANGED
|
@@ -361,10 +361,10 @@ Usage:
|
|
|
361
361
|
everystack db:backups [--stage <name>] List logical backups (id, size, created)
|
|
362
362
|
everystack db:restore --from <id> [--stage <name>] --confirm Restore a backup INTO the stage's DB (DESTRUCTIVE)
|
|
363
363
|
everystack db:backup:download <id> [--stage <name>] Presigned URL to download a backup's dump (valid 1h)
|
|
364
|
-
everystack db:export --schema <name> [--stage <name>] [--models <barrel>] Schema-scoped pg_dump
|
|
364
|
+
everystack db:export --schema <name> [--stage <name> | --database-url <url> [--out <file.dump>]] [--models <barrel>] Schema-scoped pg_dump artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp). --stage dumps the stage's private DB via the ops Lambda → S3; --database-url (explicit flag, never the env) dumps a reachable DB to a local .dump + .meta.json — the build-locally → publish → swap on-ramp
|
|
365
365
|
everystack db:swap --schema <name> --database-url <url> --from <artifact.dump> [--fingerprint <hash>] Land a schema artifact atomically: fingerprint gate → restore into <schema>_incoming (COPY-safe rewrite) → one txn (drop+rename+recreate app→schema FKs, re-apply authz) → verify → drop retiring. Refresh-free; app.* untouched. DESTRUCTIVE (--stage/--direct ops venue rides stage-write-lanes)
|
|
366
366
|
everystack db:generate [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run] Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
|
|
367
|
-
everystack db:pull [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read] Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default
|
|
367
|
+
everystack db:pull [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read] Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default. --matviews-as-tables renders every matview as defineMaterializedTable with INTROSPECTED fields (the canonical-sync flip: a pipeline-owned table everystack migrates but never refreshes) — names land in an exported materializedTables array to spread into your models; fields come back nullable/unkeyed (matviews carry no PK/NOT NULL) — tighten on review
|
|
368
368
|
Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
|
|
369
369
|
everystack db:fingerprint [--stage <name> | --database-url <url>] [--models <barrel>] [--json] Content-address the live base schema (tables+constraints+authz) and compare against the models — MATCH/MISMATCH (exit 1), plus the unfingerprinted-objects report
|
|
370
370
|
everystack db:reconcile [--stage <name> | --database-url <url>] [--apply] [--check] [--baseline] [--rebuild] [--overwrite-drift] [--json] Reconcile the derived layer (functions/views/matviews/triggers) against the DECLARED descriptors (defineView/defineMaterializedView/defineFunction/defineSql/trigger() on models, from the barrel) — the single home (db/sql is retired; leftover .sql files fail with the migration path): plan with rebuild-cost estimates by default; --check is the CI gate; --apply executes (direct connection only, atomic — DDL + provenance in one transaction) and records provenance + schema_log. Hand-edits are drift (never overwritten silently). First contact with existing objects: --baseline TRUSTS live == source (records provenance, verifies nothing), --rebuild GUARANTEES it (drop+create from source). They are mutually exclusive.
|
package/src/cli/model-render.ts
CHANGED
|
@@ -345,6 +345,16 @@ function renderField(table: TableSchema, col: ColumnSchema, known: Set<string>,
|
|
|
345
345
|
return ` ${toCamelCase(col.name)}: ${expr},${comment}`;
|
|
346
346
|
}
|
|
347
347
|
|
|
348
|
+
/**
|
|
349
|
+
* The rendered field lines for a table shape — one ` key: field.…(),` line per column,
|
|
350
|
+
* 4-space indented, comma-terminated (ready to nest under `fields: {`). Exported for the
|
|
351
|
+
* derived renderer: a matview flipped to defineMaterializedTable (--matviews-as-tables)
|
|
352
|
+
* renders its introspected columns with the SAME field vocabulary a pulled table gets.
|
|
353
|
+
*/
|
|
354
|
+
export function renderFieldLines(table: TableSchema, known: Set<string>, enums: Map<string, string[]> = new Map()): string[] {
|
|
355
|
+
return table.columns.map((c) => renderField(table, c, known, enums, new Map()));
|
|
356
|
+
}
|
|
357
|
+
|
|
348
358
|
/**
|
|
349
359
|
* The `constraints: [...]` block for a table's composite uniques, checks, indexes, and
|
|
350
360
|
* multi-column foreign keys — the table-level surface a column-scoped `field()` can't carry.
|
|
@@ -434,9 +444,12 @@ function importHeader(body: string, extra: string[] = []): string {
|
|
|
434
444
|
|
|
435
445
|
/** The barrel's module wrapper: models always; sequences/derived when the pull found any (B5). */
|
|
436
446
|
function moduleFooter(modelNames: string[], derived?: DerivedRenderResult, multiline = false): string {
|
|
447
|
+
// A materialized table (--matviews-as-tables) is a MODEL — its block rides the derived
|
|
448
|
+
// render (topo-ordered with the objects around it) but its name belongs in `models`.
|
|
449
|
+
const allModels = [...modelNames, ...(derived?.materializedTableNames ?? [])];
|
|
437
450
|
const models = multiline
|
|
438
|
-
? `export const models = [\n${
|
|
439
|
-
: `export const models = [${
|
|
451
|
+
? `export const models = [\n${allModels.map((n) => ` ${n},`).join('\n')}\n];`
|
|
452
|
+
: `export const models = [${allModels.join(', ')}];`;
|
|
440
453
|
const parts = [models];
|
|
441
454
|
const keys = ['models'];
|
|
442
455
|
if (derived?.sequenceNames.length) {
|
|
@@ -139,6 +139,31 @@ WHERE c.relkind = 'r'
|
|
|
139
139
|
ORDER BY n.nspname, c.relname, a.attnum;
|
|
140
140
|
`.trim();
|
|
141
141
|
|
|
142
|
+
/**
|
|
143
|
+
* Every column of every MATERIALIZED VIEW — same shape as COLUMNS_SQL, relkind 'm'.
|
|
144
|
+
* Only `db:pull --matviews-as-tables` runs it: the flip to defineMaterializedTable
|
|
145
|
+
* needs real fields, and the matview's columns are catalog facts the derived layer
|
|
146
|
+
* (definition-only) doesn't carry. Matviews hold no NOT NULL/defaults/PK — the rows
|
|
147
|
+
* come back nullable and unkeyed, honestly.
|
|
148
|
+
*/
|
|
149
|
+
export const MATVIEW_COLUMNS_SQL = COLUMNS_SQL.replace(`WHERE c.relkind = 'r'`, `WHERE c.relkind = 'm'`);
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Fold MATVIEW_COLUMNS_SQL rows into matview identity (`stats.player_totals`) → its
|
|
153
|
+
* columns in ordinal order — the columns map the derived renderer's table mode takes.
|
|
154
|
+
*/
|
|
155
|
+
export function matviewColumnsByIdentity(rows: ColumnRow[]): Map<string, ColumnSchema[]> {
|
|
156
|
+
const out = new Map<string, { position: number; column: ColumnSchema }[]>();
|
|
157
|
+
for (const row of rows) {
|
|
158
|
+
const d = columnRowToDescriptor(row);
|
|
159
|
+
const list = out.get(d.table) ?? [];
|
|
160
|
+
list.push({ position: d.position, column: d.column });
|
|
161
|
+
out.set(d.table, list);
|
|
162
|
+
}
|
|
163
|
+
return new Map([...out.entries()].map(([identity, cols]) =>
|
|
164
|
+
[identity, cols.sort((a, b) => a.position - b.position).map((c) => c.column)]));
|
|
165
|
+
}
|
|
166
|
+
|
|
142
167
|
/**
|
|
143
168
|
* Every table constraint — primary key, unique, check, foreign key — as its
|
|
144
169
|
* canonical `pg_get_constraintdef` text. The mapper parses that text (stable
|