@everystack/cli 0.4.9 → 0.4.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/cli/authz-contract.ts +18 -13
- package/src/cli/commands/db-export.ts +81 -0
- package/src/cli/commands/db-reconcile.ts +2 -7
- package/src/cli/commands/db-swap.ts +153 -0
- package/src/cli/derived-introspect.ts +25 -18
- package/src/cli/index.ts +10 -0
- package/src/cli/model-render.ts +72 -1
- package/src/cli/schema-fingerprint.ts +55 -17
- package/src/cli/schema-introspect.ts +15 -9
- package/src/cli/schema-rewrite.ts +67 -0
- package/src/cli/schema-swap.ts +147 -0
- package/src/cli/search-path.ts +51 -0
- package/src/cli/swap-execute.ts +133 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@everystack/cli",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.12",
|
|
4
4
|
"description": "CLI and OTA updates for Expo apps on everystack",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"author": "Scalable Technology, Inc. <licensing@scalable.technology>",
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
"structured-headers": "1.0.1",
|
|
83
83
|
"tsx": "4.21.0",
|
|
84
84
|
"typescript": "5.9.3",
|
|
85
|
-
"@everystack/model": "0.4.
|
|
85
|
+
"@everystack/model": "0.4.3"
|
|
86
86
|
},
|
|
87
87
|
"peerDependencies": {
|
|
88
88
|
"@everystack/server": ">=0.1.0",
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import { parsePgArray } from './security-catalog.js';
|
|
25
|
+
import { withCanonicalSearchPath } from './search-path.js';
|
|
25
26
|
|
|
26
27
|
// ---------------------------------------------------------------------------
|
|
27
28
|
// The contract format — the frozen, reviewable, version-controlled shape.
|
|
@@ -437,19 +438,23 @@ export async function introspectContract(
|
|
|
437
438
|
mapFunctionRow: (row: any) => { schema: string; name: string; securityDefiner: boolean; hasSearchPath: boolean },
|
|
438
439
|
functionsSql: string,
|
|
439
440
|
): Promise<AuthzContract> {
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
441
|
+
// Policy USING/WITH CHECK expressions deparse relative to the search_path — pin the
|
|
442
|
+
// canonical baseline so an ambient non-default path never reads as false authz drift.
|
|
443
|
+
return withCanonicalSearchPath(run, async () => {
|
|
444
|
+
const [rls, policies, grants, columnGrants, fnRows] = await Promise.all([
|
|
445
|
+
run(RLS_SQL),
|
|
446
|
+
run(POLICIES_SQL),
|
|
447
|
+
run(GRANTS_SQL),
|
|
448
|
+
run(COLUMN_GRANTS_SQL),
|
|
449
|
+
run(functionsSql),
|
|
450
|
+
]);
|
|
451
|
+
return assembleContract({
|
|
452
|
+
rls: rls as RlsRow[],
|
|
453
|
+
policies: policies as PolicyRow[],
|
|
454
|
+
grants: grants as GrantRow[],
|
|
455
|
+
columnGrants: columnGrants as ColumnGrantRow[],
|
|
456
|
+
functions: (fnRows as any[]).map(mapFunctionRow),
|
|
457
|
+
});
|
|
453
458
|
});
|
|
454
459
|
}
|
|
455
460
|
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `everystack db:export --schema <name> --stage <n>` — the schema-scoped, fingerprint-stamped
|
|
3
|
+
* artifact (canonical-sync gap B). pg_dump ONE schema → the stage's S3 artifact bucket, stamped
|
|
4
|
+
* with the DECLARED schema fingerprint from the checkout, so db:swap can gate an incoming artifact
|
|
5
|
+
* against the target's declared shape (declared-vs-declared: does the artifact and the stage agree
|
|
6
|
+
* on the schema's shape). everystack owns the export so that fingerprint is one formula both sides.
|
|
7
|
+
*
|
|
8
|
+
* The dump runs in the ops Lambda (the DB is private); the CLI computes the fingerprint from the
|
|
9
|
+
* models and invokes the action. Verification of the streaming path is a deploy smoke test.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
13
|
+
import { resolveConfig, opsFunction, type CliConfig } from '../config.js';
|
|
14
|
+
import { invokeAction } from '../aws.js';
|
|
15
|
+
import { fingerprintModels } from '../schema-fingerprint.js';
|
|
16
|
+
import { resolveModelsPath } from '../models-path.js';
|
|
17
|
+
import { loadModels } from './db-generate.js';
|
|
18
|
+
import { loadDeclaredDerived } from '../declared-derived.js';
|
|
19
|
+
import { pgDumpPreflightError } from './db-backup.js';
|
|
20
|
+
import { step, success, fail, info } from '../output.js';
|
|
21
|
+
|
|
22
|
+
const fmtBytes = (b?: number): string =>
|
|
23
|
+
b == null ? '?' : b >= 1024 ** 3 ? `${(b / 1024 ** 3).toFixed(1)} GB` : b >= 1024 ** 2 ? `${(b / 1024 ** 2).toFixed(1)} MB` : `${(b / 1024).toFixed(0)} KB`;
|
|
24
|
+
|
|
25
|
+
export async function dbExportCommand(flags: Record<string, string>): Promise<void> {
|
|
26
|
+
const schema = flags.schema;
|
|
27
|
+
if (!schema) {
|
|
28
|
+
fail('db:export needs --schema <name> (the schema to export as an artifact).');
|
|
29
|
+
process.exit(1);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// The DECLARED fingerprint of just this schema — the stamp the swap gate compares against.
|
|
33
|
+
const modelsPath = resolveModelsPath(flags.models);
|
|
34
|
+
let fingerprint: string;
|
|
35
|
+
try {
|
|
36
|
+
step(`Loading models from ${modelsPath}...`);
|
|
37
|
+
const models: ModelDescriptor[] = await loadModels(modelsPath);
|
|
38
|
+
const declaredDb = await loadDeclaredDerived(flags.models);
|
|
39
|
+
fingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
|
|
40
|
+
info(`Declared ${schema} fingerprint: ${fingerprint.slice(0, 12)}`);
|
|
41
|
+
} catch (err: any) {
|
|
42
|
+
fail(err.message);
|
|
43
|
+
process.exit(1);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
step('Resolving deployed config...');
|
|
47
|
+
let config: CliConfig;
|
|
48
|
+
try {
|
|
49
|
+
config = await resolveConfig(flags.stage);
|
|
50
|
+
} catch (err: any) {
|
|
51
|
+
fail(err.message);
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
info(`Region: ${config.region}, Function: ${opsFunction(config)}`);
|
|
55
|
+
|
|
56
|
+
// The dump needs the pg_dump layer — surface a missing/incompatible layer as a one-line remedy
|
|
57
|
+
// here, not a mid-dump crash (same preflight db:backup runs).
|
|
58
|
+
step('Checking the pg_dump layer...');
|
|
59
|
+
try {
|
|
60
|
+
const probe = await invokeAction(config.region, opsFunction(config), 'db:backup:probe', {});
|
|
61
|
+
const preflightError = pgDumpPreflightError(probe);
|
|
62
|
+
if (preflightError) { fail(preflightError); process.exit(1); }
|
|
63
|
+
} catch (err: any) {
|
|
64
|
+
fail(`Export pre-flight failed: ${err.message}`);
|
|
65
|
+
process.exit(1);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
step(`Running pg_dump --schema=${schema} → S3 (this may take a while for large schemas)...`);
|
|
69
|
+
let result: any;
|
|
70
|
+
try {
|
|
71
|
+
result = await invokeAction(config.region, opsFunction(config), 'db:export', { schema, stage: flags.stage, fingerprint });
|
|
72
|
+
} catch (err: any) {
|
|
73
|
+
fail(`Export failed: ${err.message}`);
|
|
74
|
+
process.exit(1);
|
|
75
|
+
}
|
|
76
|
+
if (result?.error) {
|
|
77
|
+
fail(`Export failed: ${result.error}`);
|
|
78
|
+
process.exit(1);
|
|
79
|
+
}
|
|
80
|
+
success(`Artifact ${result.id} (${fmtBytes(result.bytes)}, fingerprint ${String(result.fingerprint).slice(0, 12)}). Deploy with: everystack db:swap --schema ${schema} --stage <target>`);
|
|
81
|
+
}
|
|
@@ -179,15 +179,10 @@ export async function executeReconcile(
|
|
|
179
179
|
}
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
-
// Reset the search_path before introspecting. pg_get_viewdef renders a ref BARE while its
|
|
183
|
-
// schema is in scope and QUALIFIED once it is off — so the defHash we record must be read
|
|
184
|
-
// under the SAME (default) path every future introspection uses, or the qualified live
|
|
185
|
-
// deparse reads as drift against a bare recorded hash on the very next run. Canonical
|
|
186
|
-
// capture, always: SET LOCAL resets within the transaction; the session path otherwise.
|
|
187
|
-
if (setSearchPath) await runner(atomic ? 'SET LOCAL search_path TO DEFAULT' : 'RESET search_path');
|
|
188
|
-
|
|
189
182
|
// Re-read the catalog so provenance records the def hashes of what NOW exists, not what we hoped
|
|
190
183
|
// would exist. Inside the transaction this reads the batch's own not-yet-committed writes.
|
|
184
|
+
// introspectDerived pins the canonical search_path itself, so the defHash it records is stable
|
|
185
|
+
// regardless of the create-time wide path above (or any ambient override) — no manual reset here.
|
|
191
186
|
const after = await introspectDerived(runner);
|
|
192
187
|
const liveById = new Map(after.objects.map((o) => [o.identity, o]));
|
|
193
188
|
const srcById = new Map(parsed.objects.map((o) => [o.identity, o]));
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `everystack db:swap --schema <name> --database-url <url> --from <artifact.dump> [--confirm]` —
|
|
3
|
+
* the refresh-free, fingerprint-gated hot-swap (canonical-sync gap B).
|
|
4
|
+
*
|
|
5
|
+
* Lands a schema-scoped artifact onto a target and swaps it in atomically: gate on the declared
|
|
6
|
+
* fingerprint, snapshot, restore into `<schema>_incoming` (COPY-safe schema rewrite), the atomic
|
|
7
|
+
* rename + FK-recreate + authz-reapply transaction, a post-commit verify hook, then drop-retiring
|
|
8
|
+
* or roll back. The decision spine (swap-execute.ts) is proven on a real database; this shell wires
|
|
9
|
+
* the IO — the artifact restore and the connection.
|
|
10
|
+
*
|
|
11
|
+
* v1 venue: a direct `--database-url` (the operator connection). The credential-free `--stage`
|
|
12
|
+
* ops-Lambda / `--direct` venues ride the stage-write-lanes bricks; a multi-GB restore needs the
|
|
13
|
+
* unbounded clock those give, so `--stage` alone will refuse with the `--direct` advice.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import fs from 'node:fs';
|
|
17
|
+
import { spawn } from 'node:child_process';
|
|
18
|
+
import { Transform } from 'node:stream';
|
|
19
|
+
import { pipeline } from 'node:stream/promises';
|
|
20
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
21
|
+
import { fingerprintModels } from '../schema-fingerprint.js';
|
|
22
|
+
import { resolveModelsPath } from '../models-path.js';
|
|
23
|
+
import { loadModels } from './db-generate.js';
|
|
24
|
+
import { loadDeclaredDerived } from '../declared-derived.js';
|
|
25
|
+
import { createUrlRunner } from '../db-source.js';
|
|
26
|
+
import { executeSwap, type SwapVerdict } from '../swap-execute.js';
|
|
27
|
+
import { rewriteStatementLine, opensCopyData, closesCopyData } from '../schema-rewrite.js';
|
|
28
|
+
import { step, success, fail, warn } from '../output.js';
|
|
29
|
+
|
|
30
|
+
/** A COPY-aware line transform that rewrites the schema token on statement lines only. */
|
|
31
|
+
function schemaRewriteStream(from: string, to: string): Transform {
|
|
32
|
+
let buf = '';
|
|
33
|
+
let inCopy = false;
|
|
34
|
+
const flushLine = (line: string, push: (s: string) => void) => {
|
|
35
|
+
if (inCopy) {
|
|
36
|
+
if (closesCopyData(line)) inCopy = false;
|
|
37
|
+
push(line + '\n');
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const rewritten = rewriteStatementLine(line, from, to);
|
|
41
|
+
if (opensCopyData(rewritten)) inCopy = true;
|
|
42
|
+
push(rewritten + '\n');
|
|
43
|
+
};
|
|
44
|
+
return new Transform({
|
|
45
|
+
transform(chunk, _enc, cb) {
|
|
46
|
+
buf += chunk.toString('utf8');
|
|
47
|
+
let nl: number;
|
|
48
|
+
const out: string[] = [];
|
|
49
|
+
while ((nl = buf.indexOf('\n')) !== -1) {
|
|
50
|
+
flushLine(buf.slice(0, nl), (s) => out.push(s));
|
|
51
|
+
buf = buf.slice(nl + 1);
|
|
52
|
+
}
|
|
53
|
+
cb(null, out.join(''));
|
|
54
|
+
},
|
|
55
|
+
flush(cb) {
|
|
56
|
+
if (buf.length) flushLine(buf, (s) => this.push(s));
|
|
57
|
+
cb();
|
|
58
|
+
},
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Restore a `-Fc` artifact into `<schema>_incoming`: pg_restore -f - (archive → SQL) → the schema
|
|
64
|
+
* rewrite → psql. The archive names `<schema>`; the rewrite lands it as `<incoming>`, COPY-data-safe.
|
|
65
|
+
*/
|
|
66
|
+
async function restoreIntoIncoming(url: string, artifactPath: string, schema: string, incoming: string): Promise<void> {
|
|
67
|
+
// psql takes the connection URL directly (credential parsed by libpq, not on the process table
|
|
68
|
+
// beyond argv-as-URL); pg_restore -f - just reads the archive file to SQL on stdout.
|
|
69
|
+
const restore = spawn('pg_restore', ['-f', '-', artifactPath], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
70
|
+
const psql = spawn('psql', ['-v', 'ON_ERROR_STOP=1', '-d', url], { stdio: ['pipe', 'ignore', 'pipe'] });
|
|
71
|
+
let rErr = '', pErr = '';
|
|
72
|
+
restore.stderr.on('data', (d) => { rErr += d.toString(); });
|
|
73
|
+
psql.stderr.on('data', (d) => { pErr += d.toString(); });
|
|
74
|
+
const psqlExit = new Promise<void>((res, rej) => {
|
|
75
|
+
psql.on('error', rej);
|
|
76
|
+
psql.on('close', (c) => c === 0 ? res() : rej(new Error(`psql exited ${c}: ${pErr.trim()}`)));
|
|
77
|
+
});
|
|
78
|
+
const restoreExit = new Promise<void>((res, rej) => {
|
|
79
|
+
restore.on('error', rej);
|
|
80
|
+
restore.on('close', (c) => c === 0 ? res() : rej(new Error(`pg_restore exited ${c}: ${rErr.trim()}`)));
|
|
81
|
+
});
|
|
82
|
+
await Promise.all([
|
|
83
|
+
pipeline(restore.stdout!, schemaRewriteStream(schema, incoming), psql.stdin!),
|
|
84
|
+
restoreExit,
|
|
85
|
+
psqlExit,
|
|
86
|
+
]);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Load the artifact's meta (fingerprint) from a sibling `.meta.json`, or fall back to a flag. */
|
|
90
|
+
function readArtifactFingerprint(artifactPath: string, flag?: string): string | null {
|
|
91
|
+
if (flag) return flag;
|
|
92
|
+
const metaPath = artifactPath.replace(/\.dump(\.gz)?$/, '.meta.json');
|
|
93
|
+
try {
|
|
94
|
+
return JSON.parse(fs.readFileSync(metaPath, 'utf8')).fingerprint ?? null;
|
|
95
|
+
} catch {
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function dbSwapCommand(flags: Record<string, string>): Promise<void> {
|
|
101
|
+
const schema = flags.schema;
|
|
102
|
+
const url = flags['database-url'] || process.env.DATABASE_URL;
|
|
103
|
+
const from = flags.from;
|
|
104
|
+
if (!schema) { fail('db:swap needs --schema <name>.'); process.exit(1); }
|
|
105
|
+
if (!from) { fail('db:swap needs --from <artifact.dump> (the schema-scoped -Fc archive to land).'); process.exit(1); }
|
|
106
|
+
if (!url) {
|
|
107
|
+
fail('db:swap v1 needs a direct --database-url (the operator connection). The credential-free --stage venue rides the stage-write-lanes bricks; a large restore needs its unbounded clock.');
|
|
108
|
+
process.exit(1);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const modelsPath = resolveModelsPath(flags.models);
|
|
112
|
+
let models: ModelDescriptor[];
|
|
113
|
+
let declaredFingerprint: string;
|
|
114
|
+
try {
|
|
115
|
+
step(`Loading models from ${modelsPath}...`);
|
|
116
|
+
models = await loadModels(modelsPath);
|
|
117
|
+
const declaredDb = await loadDeclaredDerived(flags.models);
|
|
118
|
+
declaredFingerprint = fingerprintModels(models, { schemas: [schema], sequences: declaredDb?.sequences }).hash;
|
|
119
|
+
} catch (err: any) { fail(err.message); process.exit(1); }
|
|
120
|
+
|
|
121
|
+
const artifactFingerprint = readArtifactFingerprint(from, flags.fingerprint);
|
|
122
|
+
if (!artifactFingerprint) {
|
|
123
|
+
fail(`db:swap can't find the artifact fingerprint — expected a sibling .meta.json next to ${from}, or pass --fingerprint. Without it the gate can't run, and an unchecked swap is exactly what the gate prevents.`);
|
|
124
|
+
process.exit(1);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const { runner, end } = await createUrlRunner(url);
|
|
128
|
+
try {
|
|
129
|
+
step(`Swapping ${schema} — gate, land incoming, atomic swap, verify...`);
|
|
130
|
+
const res = await executeSwap(runner, {
|
|
131
|
+
models, schema,
|
|
132
|
+
artifactFingerprint,
|
|
133
|
+
declaredFingerprint,
|
|
134
|
+
applyIncoming: async () => { await restoreIntoIncoming(url, from, schema, `${schema}_incoming`); },
|
|
135
|
+
// Snapshot + verify-hook wiring land with the ops venue; a direct v1 swap warns rather than
|
|
136
|
+
// silently skipping the safety net.
|
|
137
|
+
snapshot: async () => { warn('no snapshot taken (direct v1) — take one first: everystack db:backup --database-url … before a production swap.'); },
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
if (res.status === 'swapped') {
|
|
141
|
+
success(`Swapped ${schema} — the artifact is live (no refresh ran).`);
|
|
142
|
+
for (const w of res.warnings ?? []) warn(`verify warning: ${w.name}${w.detail ? ` — ${w.detail}` : ''}`);
|
|
143
|
+
} else {
|
|
144
|
+
fail(`db:swap ${res.status}: ${res.reason}`);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
}
|
|
147
|
+
} catch (err: any) {
|
|
148
|
+
fail(`Swap failed: ${err.message}`);
|
|
149
|
+
process.exit(1);
|
|
150
|
+
} finally {
|
|
151
|
+
await end?.();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
import { createHash } from 'node:crypto';
|
|
21
21
|
import { IGNORED_SCHEMAS, type QueryRunner } from './authz-contract.js';
|
|
22
22
|
import { normalizeSql, type DerivedKind } from './derived-source.js';
|
|
23
|
+
import { withCanonicalSearchPath } from './search-path.js';
|
|
23
24
|
|
|
24
25
|
export interface LiveObject {
|
|
25
26
|
kind: DerivedKind;
|
|
@@ -529,23 +530,29 @@ export function assembleDerivedCatalog(rows: DerivedRows): DerivedCatalog {
|
|
|
529
530
|
* the reconciler has never touched) folds to an empty claims list.
|
|
530
531
|
*/
|
|
531
532
|
export async function introspectDerived(run: QueryRunner): Promise<DerivedCatalog> {
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
533
|
+
// View/matview/function bodies deparse relative to the search_path (`pg_get_viewdef`
|
|
534
|
+
// renders a ref bare while its schema is in scope, qualified once off). Pin the
|
|
535
|
+
// canonical baseline so the recorded def hash is stable across runs and paths — this
|
|
536
|
+
// also subsumes the reconcile apply's pre-introspection reset. See search-path.ts.
|
|
537
|
+
return withCanonicalSearchPath(run, async () => {
|
|
538
|
+
const [relations, functions, indexes, depends, triggers, grants] = await Promise.all([
|
|
539
|
+
run(DERIVED_RELATIONS_SQL), run(DERIVED_FUNCTIONS_SQL), run(MATVIEW_INDEXES_SQL), run(DERIVED_DEPENDS_SQL),
|
|
540
|
+
run(DERIVED_TRIGGERS_SQL), run(DERIVED_GRANTS_SQL),
|
|
541
|
+
]);
|
|
542
|
+
let provenance: ProvenanceRawRow[] = [];
|
|
543
|
+
try {
|
|
544
|
+
provenance = (await run(PROVENANCE_SQL)) as ProvenanceRawRow[];
|
|
545
|
+
} catch {
|
|
546
|
+
// everystack.derived_provenance does not exist yet — nothing has been reconciled.
|
|
547
|
+
}
|
|
548
|
+
return assembleDerivedCatalog({
|
|
549
|
+
relations: relations as RelationRow[],
|
|
550
|
+
functions: functions as FunctionRow[],
|
|
551
|
+
indexes: indexes as IndexDefRow[],
|
|
552
|
+
depends: depends as DependsRow[],
|
|
553
|
+
triggers: triggers as TriggerRow[],
|
|
554
|
+
grants: grants as GrantAclRow[],
|
|
555
|
+
provenance,
|
|
556
|
+
});
|
|
550
557
|
});
|
|
551
558
|
}
|
package/src/cli/index.ts
CHANGED
|
@@ -25,6 +25,8 @@ import { dbTemplateRefreshCommand, dbBranchCommand } from './commands/db-branch.
|
|
|
25
25
|
import { dbForkCommand } from './commands/db-fork.js';
|
|
26
26
|
import { dbSnapshotCommand, dbSnapshotsCommand } from './commands/db-snapshot.js';
|
|
27
27
|
import { dbBackupProbeCommand, dbBackupCommand, dbBackupsCommand, dbRestoreCommand, dbBackupDownloadCommand } from './commands/db-backup.js';
|
|
28
|
+
import { dbExportCommand } from './commands/db-export.js';
|
|
29
|
+
import { dbSwapCommand } from './commands/db-swap.js';
|
|
28
30
|
import { consoleCommand } from './commands/console.js';
|
|
29
31
|
import { cachePurgeCommand } from './commands/cache.js';
|
|
30
32
|
import { statusCommand } from './commands/status.js';
|
|
@@ -181,6 +183,12 @@ async function main() {
|
|
|
181
183
|
case 'db:restore':
|
|
182
184
|
await dbRestoreCommand(flags);
|
|
183
185
|
break;
|
|
186
|
+
case 'db:export':
|
|
187
|
+
await dbExportCommand(flags);
|
|
188
|
+
break;
|
|
189
|
+
case 'db:swap':
|
|
190
|
+
await dbSwapCommand(flags);
|
|
191
|
+
break;
|
|
184
192
|
case 'db:backup:download': {
|
|
185
193
|
const backupId = args[1] && !args[1].startsWith('--') ? args[1] : undefined;
|
|
186
194
|
await dbBackupDownloadCommand(flags, backupId);
|
|
@@ -353,6 +361,8 @@ Usage:
|
|
|
353
361
|
everystack db:backups [--stage <name>] List logical backups (id, size, created)
|
|
354
362
|
everystack db:restore --from <id> [--stage <name>] --confirm Restore a backup INTO the stage's DB (DESTRUCTIVE)
|
|
355
363
|
everystack db:backup:download <id> [--stage <name>] Presigned URL to download a backup's dump (valid 1h)
|
|
364
|
+
everystack db:export --schema <name> [--stage <name>] [--models <barrel>] Schema-scoped pg_dump → S3 artifact, stamped with the DECLARED schema fingerprint (the canonical-sync export; db:swap gates on that stamp)
|
|
365
|
+
everystack db:swap --schema <name> --database-url <url> --from <artifact.dump> [--fingerprint <hash>] Land a schema artifact atomically: fingerprint gate → restore into <schema>_incoming (COPY-safe rewrite) → one txn (drop+rename+recreate app→schema FKs, re-apply authz) → verify → drop retiring. Refresh-free; app.* untouched. DESTRUCTIVE (--stage/--direct ops venue rides stage-write-lanes)
|
|
356
366
|
everystack db:generate [--stage <name> | --database-url <url>] [--name <label>] [--models db/models/index.ts] [--schema-out db/schema.generated.ts] [--allow-drops] [--apply] [--dry-run] Diff models vs the live DB → next migration file, or with --apply execute it directly (one transaction, schema_log recorded, verified by re-diff — no drizzle folder needed; direct connection only; DROPs held back unless --allow-drops). --dry-run prints the edge and writes NOTHING (no migration, no journal entry, no schema refresh) — the preview verb; db:diff computes a models-vs-models edge with no database at all. The resolved --schema-out is recorded in the migration journal: later flag-less runs reuse it (flag > recorded > default), a differing flag updates the record and says so
|
|
357
367
|
everystack db:pull [--stage <name> | --database-url <url>] [--schema public] [--out <dir | file.ts>] [--derived-out <file.ts>] [--abilities public-read] Introspect the live DB → render field() Models (the brownfield on-ramp). --out <dir> writes one file per model + index.ts (the default shape); --out <file.ts> writes a single module; stdout otherwise. --derived-out <file.ts> extracts the derived layer (descriptors + sequences) as its own self-contained module — alone it leaves the models untouched (the hand-maintained-barrel splice); with --out the models render omits the now-external derived layer. Every model scaffolds its authz decision as comments (db:check fails until authored); --abilities public-read stamps the common stanza (public read, admin write) uncommented — explicit generated code, never a runtime default
|
|
358
368
|
Both introspect via the deployed ops Lambda by default; --database-url (or an inherited DATABASE_URL) connects directly — for a schema that exists only on a local Postgres.
|
package/src/cli/model-render.ts
CHANGED
|
@@ -208,6 +208,77 @@ function renderDefault(expr: string): string | null {
|
|
|
208
208
|
return null;
|
|
209
209
|
}
|
|
210
210
|
|
|
211
|
+
const NUMERIC_ARRAY_BASE = /^(smallint|integer|bigint|numeric|decimal|real|double precision)\b/;
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* The `.default([...])` modifier for an ARRAY column — the spelling the array-default
|
|
215
|
+
* guard requires (a string default on an array field throws; `.default("{}")` was the
|
|
216
|
+
* old, now-rejected pull output). Parses the Postgres array literal back into the JS
|
|
217
|
+
* array the compiler round-trips from: `'{}'` → `[]`, `'{"a","b"}'` → `['a', 'b']`,
|
|
218
|
+
* `'{1,2}'` → `[1, 2]`. Returns null for anything it can't confidently invert (nested
|
|
219
|
+
* arrays, NULL elements, an `ARRAY[...]`/function expression) — the caller falls back to
|
|
220
|
+
* `.defaultSql()`, which round-trips via the raw SQL (the diff normalizes both sides).
|
|
221
|
+
*/
|
|
222
|
+
function renderArrayDefault(expr: string, arrayType: string): string | null {
|
|
223
|
+
const n = normalizeDefault(expr);
|
|
224
|
+
if (n == null) return null;
|
|
225
|
+
const lit = n.match(/^'(\{[\s\S]*\})'$/);
|
|
226
|
+
if (!lit) return null; // not a `'{…}'` literal (e.g. an ARRAY[…] expression) → defaultSql
|
|
227
|
+
const elems = parsePgArrayElements(lit[1]);
|
|
228
|
+
if (elems == null) return null;
|
|
229
|
+
const base = arrayType.slice(0, -2);
|
|
230
|
+
const numeric = NUMERIC_ARRAY_BASE.test(base);
|
|
231
|
+
const boolean = base === 'boolean';
|
|
232
|
+
const rendered = elems.map((e) => {
|
|
233
|
+
// A bare NULL is a real NULL element (a quoted "NULL" is the string) — we don't model it.
|
|
234
|
+
if (!e.quoted && e.raw.toUpperCase() === 'NULL') return null;
|
|
235
|
+
if (boolean) return e.raw === 'true' || e.raw === 'false' ? e.raw : null;
|
|
236
|
+
if (numeric) return /^-?\d+(\.\d+)?$/.test(e.raw) ? e.raw : null;
|
|
237
|
+
return tsLiteral(e.raw);
|
|
238
|
+
});
|
|
239
|
+
if (rendered.some((r) => r == null)) return null;
|
|
240
|
+
return `.default([${rendered.join(', ')}])`;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Split the body of a Postgres array literal (`{…}`, braces included) into its top-level
|
|
245
|
+
* elements, each tagged quoted-or-bare with backslash escapes resolved. Returns `[]` for
|
|
246
|
+
* `{}`, or null for a nested array or a malformed literal (the caller then falls back).
|
|
247
|
+
*/
|
|
248
|
+
function parsePgArrayElements(literal: string): Array<{ raw: string; quoted: boolean }> | null {
|
|
249
|
+
const body = literal.slice(1, -1);
|
|
250
|
+
if (body === '') return [];
|
|
251
|
+
const out: Array<{ raw: string; quoted: boolean }> = [];
|
|
252
|
+
let i = 0;
|
|
253
|
+
while (i < body.length) {
|
|
254
|
+
let raw = '';
|
|
255
|
+
let quoted = false;
|
|
256
|
+
if (body[i] === '{') return null; // nested array — not modeled
|
|
257
|
+
if (body[i] === '"') {
|
|
258
|
+
quoted = true;
|
|
259
|
+
i++;
|
|
260
|
+
while (i < body.length && body[i] !== '"') {
|
|
261
|
+
if (body[i] === '\\') { raw += body[i + 1] ?? ''; i += 2; continue; }
|
|
262
|
+
raw += body[i]; i++;
|
|
263
|
+
}
|
|
264
|
+
if (body[i] !== '"') return null; // unterminated quote
|
|
265
|
+
i++;
|
|
266
|
+
} else {
|
|
267
|
+
while (i < body.length && body[i] !== ',') {
|
|
268
|
+
if (body[i] === '{' || body[i] === '"') return null;
|
|
269
|
+
raw += body[i]; i++;
|
|
270
|
+
}
|
|
271
|
+
raw = raw.trim();
|
|
272
|
+
}
|
|
273
|
+
out.push({ raw, quoted });
|
|
274
|
+
if (i < body.length) {
|
|
275
|
+
if (body[i] !== ',') return null; // trailing junk after a quoted element
|
|
276
|
+
i++;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
return out;
|
|
280
|
+
}
|
|
281
|
+
|
|
211
282
|
/** A single-quoted TS string literal (the model idiom), escaping embedded quotes. */
|
|
212
283
|
function tsLiteral(value: string): string {
|
|
213
284
|
return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
|
|
@@ -260,7 +331,7 @@ function renderField(table: TableSchema, col: ColumnSchema, known: Set<string>,
|
|
|
260
331
|
}
|
|
261
332
|
|
|
262
333
|
if (col.default != null && !serial) {
|
|
263
|
-
const d = renderDefault(col.default);
|
|
334
|
+
const d = col.type.endsWith('[]') ? renderArrayDefault(col.default, col.type) : renderDefault(col.default);
|
|
264
335
|
// No builder maps it → declare it verbatim with the escape hatch. RAW, not the
|
|
265
336
|
// normalized form — stripping a trailing cast here could change semantics
|
|
266
337
|
// (`'7 days'::interval`); the diff normalizes both sides for comparison instead.
|
|
@@ -161,26 +161,54 @@ export interface CanonicalState {
|
|
|
161
161
|
sequences?: Array<Record<string, unknown>>;
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
/** The schema of a (possibly) qualified name — `stats.entities` → `stats`, a bare name → `public`. */
|
|
165
|
+
export function schemaOf(qualifiedName: string): string {
|
|
166
|
+
const dot = qualifiedName.indexOf('.');
|
|
167
|
+
return dot === -1 ? 'public' : qualifiedName.slice(0, dot);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export interface CanonicalizeOptions {
|
|
171
|
+
/** Restrict the state to these schemas — the content address of ONE schema (e.g. a
|
|
172
|
+
* schema-scoped artifact) instead of the whole database. Omitted = every schema. */
|
|
173
|
+
schemas?: string[];
|
|
174
|
+
}
|
|
175
|
+
|
|
164
176
|
export function canonicalizeState(
|
|
165
177
|
snapshot: SchemaSnapshot,
|
|
166
178
|
authzTables: TableContract[],
|
|
179
|
+
opts: CanonicalizeOptions = {},
|
|
167
180
|
): CanonicalState {
|
|
181
|
+
const inScope = opts.schemas ? new Set(opts.schemas) : null;
|
|
182
|
+
const keepTable = (name: string) => inScope === null || inScope.has(schemaOf(name));
|
|
183
|
+
|
|
184
|
+
const tables = snapshot.tables.filter((t) => keepTable(t.table));
|
|
185
|
+
// Enums carry no schema in the descriptor (keyed by bare name, like the model side).
|
|
186
|
+
// Scope them by USE: an enum is in-scope when a kept table has a column of that type
|
|
187
|
+
// (the type may be qualified under canonical introspection — `stats.status` — so match
|
|
188
|
+
// on the bare tail). This keeps the exact set the scoped models declare.
|
|
189
|
+
const usedEnum = new Set<string>();
|
|
190
|
+
for (const t of tables) for (const c of t.columns) usedEnum.add(schemaOf(c.type) === 'public' ? c.type : c.type.slice(c.type.indexOf('.') + 1));
|
|
191
|
+
|
|
168
192
|
const sequences = byKey(
|
|
169
|
-
(snapshot.sequences ?? [])
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
193
|
+
(snapshot.sequences ?? [])
|
|
194
|
+
.filter((s) => inScope === null || inScope.has(schemaOf(s.name)))
|
|
195
|
+
.map((s) => ({
|
|
196
|
+
name: s.name, as: s.as,
|
|
197
|
+
...(s.start != null ? { start: s.start } : {}),
|
|
198
|
+
...(s.increment != null ? { increment: s.increment } : {}),
|
|
199
|
+
})),
|
|
174
200
|
(s) => String(s.name),
|
|
175
201
|
);
|
|
176
202
|
return {
|
|
177
203
|
v: FINGERPRINT_VERSION,
|
|
178
|
-
tables: byKey(
|
|
204
|
+
tables: byKey(tables.map(canonicalTable), (t) => String(t.table)),
|
|
179
205
|
enums: byKey(
|
|
180
|
-
(snapshot.enums ?? [])
|
|
206
|
+
(snapshot.enums ?? [])
|
|
207
|
+
.filter((e) => inScope === null || usedEnum.has(e.name))
|
|
208
|
+
.map((e) => ({ name: e.name, values: e.values })),
|
|
181
209
|
(e) => e.name,
|
|
182
210
|
),
|
|
183
|
-
authz: byKey(authzTables.map(canonicalAuthzTable), (t) => String(t.table)),
|
|
211
|
+
authz: byKey(authzTables.filter((t) => keepTable(t.table)).map(canonicalAuthzTable), (t) => String(t.table)),
|
|
184
212
|
...(sequences.length ? { sequences } : {}),
|
|
185
213
|
};
|
|
186
214
|
}
|
|
@@ -195,28 +223,38 @@ export interface Fingerprint {
|
|
|
195
223
|
export function fingerprintState(
|
|
196
224
|
snapshot: SchemaSnapshot,
|
|
197
225
|
authzTables: TableContract[],
|
|
226
|
+
opts: CanonicalizeOptions = {},
|
|
198
227
|
): Fingerprint {
|
|
199
|
-
const canonical = canonicalizeState(snapshot, authzTables);
|
|
228
|
+
const canonical = canonicalizeState(snapshot, authzTables, opts);
|
|
200
229
|
return { hash: createHash('sha256').update(stableStringify(canonical)).digest('hex'), canonical };
|
|
201
230
|
}
|
|
202
231
|
|
|
203
|
-
/** The DECLARED state: compile the models to the same canonical shape and hash it.
|
|
232
|
+
/** The DECLARED state: compile the models to the same canonical shape and hash it. When
|
|
233
|
+
* `schemas` is set, only models in those schemas are compiled — the content address of a
|
|
234
|
+
* schema slice (the export stamp and the swap gate compute this identically). */
|
|
204
235
|
export function fingerprintModels(
|
|
205
236
|
models: ModelDescriptor[],
|
|
206
|
-
opts: { schema?: string; sequences?: SequenceDescriptor[] } = {},
|
|
237
|
+
opts: { schema?: string; sequences?: SequenceDescriptor[]; schemas?: string[] } = {},
|
|
207
238
|
): Fingerprint {
|
|
239
|
+
const scoped = opts.schemas ? models.filter((m) => opts.schemas!.includes(m.schema)) : models;
|
|
208
240
|
const snapshot: SchemaSnapshot = {
|
|
209
|
-
tables:
|
|
210
|
-
enums: compileEnums(
|
|
241
|
+
tables: scoped.map((m) => compileTableSchema(m, opts)),
|
|
242
|
+
enums: compileEnums(scoped),
|
|
211
243
|
...(opts.sequences?.length ? { sequences: compileSequences(opts.sequences) } : {}),
|
|
212
244
|
};
|
|
213
|
-
const authzTables =
|
|
214
|
-
|
|
245
|
+
const authzTables = scoped.map((m) => compileTableContract(m, opts));
|
|
246
|
+
// The models are already scoped, so no further schema filter — but standalone sequences
|
|
247
|
+
// are declared apart from models; scope them here too.
|
|
248
|
+
return fingerprintState(snapshot, authzTables, opts.schemas ? { schemas: opts.schemas } : {});
|
|
215
249
|
}
|
|
216
250
|
|
|
217
251
|
/** Convenience: the live side, from the two existing introspections. */
|
|
218
|
-
export function fingerprintLive(
|
|
219
|
-
|
|
252
|
+
export function fingerprintLive(
|
|
253
|
+
snapshot: SchemaSnapshot,
|
|
254
|
+
contract: AuthzContract,
|
|
255
|
+
opts: CanonicalizeOptions = {},
|
|
256
|
+
): Fingerprint {
|
|
257
|
+
return fingerprintState(snapshot, contract.tables, opts);
|
|
220
258
|
}
|
|
221
259
|
|
|
222
260
|
// ---------------------------------------------------------------------------
|
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
16
|
import { IGNORED_SCHEMAS, coerceBool, type QueryRunner } from './authz-contract.js';
|
|
17
|
+
import { withCanonicalSearchPath } from './search-path.js';
|
|
17
18
|
|
|
18
19
|
// ---------------------------------------------------------------------------
|
|
19
20
|
// The data-layer snapshot — the structured shape both producers target.
|
|
@@ -579,14 +580,19 @@ export function assembleSchema(rows: SchemaRows): SchemaSnapshot {
|
|
|
579
580
|
* `assembleSchema`. This is the `current` side `db:generate` diffs the compiled Models against.
|
|
580
581
|
*/
|
|
581
582
|
export async function introspectSchema(run: QueryRunner): Promise<SchemaSnapshot> {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
return
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
583
|
+
// Pin the canonical search_path: column defaults, CHECK constraints, and index
|
|
584
|
+
// expressions all deparse relative to it, so an ambient non-default path would
|
|
585
|
+
// read as false drift. See search-path.ts.
|
|
586
|
+
return withCanonicalSearchPath(run, async () => {
|
|
587
|
+
const [columns, constraints, enums, indexes, sequences] = await Promise.all([
|
|
588
|
+
run(COLUMNS_SQL), run(CONSTRAINTS_SQL), run(ENUMS_SQL), run(INDEXES_SQL), run(SEQUENCES_SQL),
|
|
589
|
+
]);
|
|
590
|
+
return assembleSchema({
|
|
591
|
+
columns: columns as ColumnRow[],
|
|
592
|
+
constraints: constraints as ConstraintRow[],
|
|
593
|
+
enums: enums as EnumRow[],
|
|
594
|
+
indexes: indexes as IndexRow[],
|
|
595
|
+
sequences: sequences as SequenceRow[],
|
|
596
|
+
});
|
|
591
597
|
});
|
|
592
598
|
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* schema-rewrite — land a schema dump under a DIFFERENT schema name, COPY-data-safe.
|
|
3
|
+
*
|
|
4
|
+
* A `-Fc` archive pins the schema name it was dumped from; `pg_restore` filters by schema but can't
|
|
5
|
+
* remap one. For a ZERO-WINDOW hot-swap the incoming data must land in `<schema>_incoming` while the
|
|
6
|
+
* live `<schema>` keeps serving, so we restore the archive to SQL (`pg_restore -f -`) and rewrite the
|
|
7
|
+
* schema token on the way to `psql`. everystack owns both the export and this rewrite, so the token
|
|
8
|
+
* is our own dump output — deterministic.
|
|
9
|
+
*
|
|
10
|
+
* The one hazard is a COPY data row whose VALUE contains the schema name (`see stats.entities for…`):
|
|
11
|
+
* it must survive untouched. So the rewrite is COPY-aware — it rewrites STATEMENT lines only and
|
|
12
|
+
* passes every byte between `COPY … FROM stdin;` and the terminating `\.` through verbatim.
|
|
13
|
+
*
|
|
14
|
+
* Pure and line-oriented so it composes into a stream transform (the ops/-direct restore pipes
|
|
15
|
+
* pg_restore stdout → this → psql stdin); a failure here is a loud restore error, never silent
|
|
16
|
+
* corruption. When the rewrite can't be trusted (a truly exotic dump), the swap's documented
|
|
17
|
+
* fallback is the rename-dance (a brief window, snapshot-backed).
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
const IDENT = '[A-Za-z_][A-Za-z0-9_$]*';
|
|
21
|
+
|
|
22
|
+
/** True once this line OPENS a COPY data block (`COPY … FROM stdin;`) — data follows until `\.`. */
|
|
23
|
+
export function opensCopyData(line: string): boolean {
|
|
24
|
+
return /^\s*COPY\s+.*\sFROM\s+stdin;\s*$/i.test(line);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** True when this line CLOSES a COPY data block (a lone `\.`). */
|
|
28
|
+
export function closesCopyData(line: string): boolean {
|
|
29
|
+
return line === '\\.' || /^\\\.\s*$/.test(line);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Rewrite the schema token `from` → `to` on a single STATEMENT line (never call on COPY data).
|
|
34
|
+
* Handles the forms pg_dump emits: the qualifier (`from.x`, `"from".x`), the standalone schema
|
|
35
|
+
* statements (`CREATE SCHEMA from`, `ALTER SCHEMA from …`, `DROP SCHEMA … from`), and a
|
|
36
|
+
* `SET search_path` naming it. The schema token must be a plain identifier at both ends so a
|
|
37
|
+
* substring of another name (`from_archive`) is never touched.
|
|
38
|
+
*/
|
|
39
|
+
export function rewriteStatementLine(line: string, from: string, to: string): string {
|
|
40
|
+
const f = from.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
41
|
+
// 1. Schema-qualified refs: `from.` or `"from".` → `to.` (normalize to bare; the token is safe).
|
|
42
|
+
let out = line.replace(new RegExp(`(^|[^A-Za-z0-9_$."])(?:${f}|"${f}")\\.`, 'g'), `$1${to}.`);
|
|
43
|
+
// 2. Standalone schema in CREATE/ALTER/DROP SCHEMA and search_path — the token as a whole word,
|
|
44
|
+
// bare or quoted, not followed by a dot (those were handled above).
|
|
45
|
+
out = out.replace(new RegExp(`(^|[^A-Za-z0-9_$.])(?:${f}|"${f}")(?![A-Za-z0-9_$."])`, 'g'), `$1${to}`);
|
|
46
|
+
return out;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Rewrite a whole plain-SQL dump, COPY-data-safe. Splits on newlines, tracks COPY blocks, and
|
|
51
|
+
* rewrites only statement lines. Returns the rewritten SQL. (The streaming transform in the restore
|
|
52
|
+
* pipeline applies `rewriteStatementLine` per line with the same COPY tracking; this is the
|
|
53
|
+
* in-memory form the tests pin.)
|
|
54
|
+
*/
|
|
55
|
+
export function rewriteSchemaDump(sql: string, from: string, to: string): string {
|
|
56
|
+
const lines = sql.split('\n');
|
|
57
|
+
let inCopy = false;
|
|
58
|
+
for (let i = 0; i < lines.length; i++) {
|
|
59
|
+
if (inCopy) {
|
|
60
|
+
if (closesCopyData(lines[i])) inCopy = false;
|
|
61
|
+
continue; // data (or the terminator) — never rewritten
|
|
62
|
+
}
|
|
63
|
+
lines[i] = rewriteStatementLine(lines[i], from, to);
|
|
64
|
+
if (opensCopyData(lines[i])) inCopy = true;
|
|
65
|
+
}
|
|
66
|
+
return lines.join('\n');
|
|
67
|
+
}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* schema-swap — the refresh-free, FK-safe, atomic hot-swap of one schema (the core of
|
|
3
|
+
* db:swap; docs/plans/canonical-sync-artifacts.md, gap B).
|
|
4
|
+
*
|
|
5
|
+
* The incoming data has already been restored into `<schema>_incoming` while the live
|
|
6
|
+
* `<schema>` keeps serving. This renders the ONE transaction that swaps them in atomically:
|
|
7
|
+
* readers see the old schema or the new, never a partial. Pure — no database here; the
|
|
8
|
+
* command sends `statements` as one transaction and owns the snapshot / verify / drop-retiring
|
|
9
|
+
* lifecycle around it.
|
|
10
|
+
*
|
|
11
|
+
* The order is load-bearing, and every step verified on PostgreSQL 16:
|
|
12
|
+
* 1. DROP the app→schema foreign keys. They bind their target by OID, so the rename below
|
|
13
|
+
* would carry them onto the RETIRING schema (they would reference stale data, and the
|
|
14
|
+
* final DROP of the retiring schema would fail). Drop them first.
|
|
15
|
+
* 2. RENAME the live schema out (`<schema>` → `<schema>_retiring`) and the incoming in
|
|
16
|
+
* (`<schema>_incoming` → `<schema>`). Schema renames are transactional and atomic.
|
|
17
|
+
* 3. RECREATE the app→schema foreign keys against the NEW schema. This RE-VALIDATES against
|
|
18
|
+
* the new data: an app row referencing a key the new artifact dropped fails the
|
|
19
|
+
* ADD CONSTRAINT, the whole transaction rolls back, and the swap is refused — the
|
|
20
|
+
* cross-schema integrity gate, for free.
|
|
21
|
+
* 4. RE-APPLY the schema's authz (RLS, policies, grants) from the declared descriptors. The
|
|
22
|
+
* incoming schema was restored `--no-owner --no-privileges`, so it carries no grants;
|
|
23
|
+
* GRANT/POLICY are transactional, so they ride the same transaction — there is no
|
|
24
|
+
* committed instant where the new schema serves with absent or stale grants.
|
|
25
|
+
*
|
|
26
|
+
* No REFRESH anywhere: the artifact ships computed rows as tables (defineMaterializedTable),
|
|
27
|
+
* so the swap is a pointer flip, never a recompute.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
31
|
+
import { modelForeignKeys, qualifiedTable } from './schema-compile.js';
|
|
32
|
+
import { compileTableContract } from './authz-compile.js';
|
|
33
|
+
import { emitReconcileSql } from './authz-reconcile.js';
|
|
34
|
+
import { schemaOf } from './schema-fingerprint.js';
|
|
35
|
+
|
|
36
|
+
const SAFE_SCHEMA = /^[a-z_][a-z0-9_$]*$/;
|
|
37
|
+
|
|
38
|
+
function assertSafeSchema(name: string): void {
|
|
39
|
+
if (!SAFE_SCHEMA.test(name)) {
|
|
40
|
+
throw new Error(`swap schema ${JSON.stringify(name)} is not a plain lowercase identifier — everystack swaps schemas it can name without quoting games.`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** ` ON DELETE … ON UPDATE …` for a FK, or '' when both are the PostgreSQL default. */
|
|
45
|
+
function refActionSql(fk: { onDelete?: string; onUpdate?: string }): string {
|
|
46
|
+
let s = '';
|
|
47
|
+
if (fk.onDelete) s += ` ON DELETE ${fk.onDelete.toUpperCase()}`;
|
|
48
|
+
if (fk.onUpdate) s += ` ON UPDATE ${fk.onUpdate.toUpperCase()}`;
|
|
49
|
+
return s;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Quote a (possibly) schema-qualified ref: `stats.entities` → `"stats"."entities"`, `uploads` → `"uploads"`. */
|
|
53
|
+
function quoteRef(name: string): string {
|
|
54
|
+
return name.split('.').map((p) => `"${p}"`).join('.');
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export interface CrossSchemaFk {
|
|
58
|
+
/** The model that holds the FK (an app/public table pointing INTO the swapped schema). */
|
|
59
|
+
table: string;
|
|
60
|
+
name: string;
|
|
61
|
+
dropSql: string;
|
|
62
|
+
addSql: string;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* The foreign keys that cross INTO `targetSchema` from a model in ANOTHER schema — the only FKs
|
|
67
|
+
* the swap must drop and recreate. FKs WITHIN the swapped schema ride the dump; FKs from the
|
|
68
|
+
* swapped schema OUT to a stable schema (e.g. stats → app) resolve by name on restore and are
|
|
69
|
+
* untouched. Only external → swapped edges bind the old schema by OID.
|
|
70
|
+
*/
|
|
71
|
+
export function crossSchemaForeignKeys(models: ModelDescriptor[], targetSchema: string): CrossSchemaFk[] {
|
|
72
|
+
const out: CrossSchemaFk[] = [];
|
|
73
|
+
for (const model of models) {
|
|
74
|
+
if ((model.schema || 'public') === targetSchema) continue; // internal edge, not external
|
|
75
|
+
const from = qualifiedTable(model);
|
|
76
|
+
for (const fk of modelForeignKeys(model)) {
|
|
77
|
+
if (schemaOf(fk.refTable) !== targetSchema) continue; // not pointing into the swapped schema
|
|
78
|
+
const cols = fk.columns.map((c) => `"${c}"`).join(', ');
|
|
79
|
+
const refCols = fk.refColumns.map((c) => `"${c}"`).join(', ');
|
|
80
|
+
out.push({
|
|
81
|
+
table: from,
|
|
82
|
+
name: fk.name,
|
|
83
|
+
dropSql: `ALTER TABLE ${from} DROP CONSTRAINT IF EXISTS "${fk.name}";`,
|
|
84
|
+
addSql: `ALTER TABLE ${from} ADD CONSTRAINT "${fk.name}" FOREIGN KEY (${cols}) REFERENCES ${quoteRef(fk.refTable)}(${refCols})${refActionSql(fk)};`,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
export interface SwapPlan {
|
|
92
|
+
/** The ordered SQL for ONE transaction (the command wraps BEGIN/COMMIT). */
|
|
93
|
+
statements: string[];
|
|
94
|
+
/** The app→schema FKs dropped and recreated (the integrity gate). */
|
|
95
|
+
crossSchemaFks: CrossSchemaFk[];
|
|
96
|
+
/** The schema the live data is renamed to before the retiring drop (outside the txn). */
|
|
97
|
+
retiring: string;
|
|
98
|
+
incoming: string;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export interface SwapOptions {
|
|
102
|
+
/** The schema being swapped (e.g. `stats`). */
|
|
103
|
+
schema: string;
|
|
104
|
+
/** Where the incoming data was restored. Default `<schema>_incoming`. */
|
|
105
|
+
incoming?: string;
|
|
106
|
+
/** Where the live schema is renamed. Default `<schema>_retiring`; pass a stamped name for uniqueness. */
|
|
107
|
+
retiring?: string;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Render the atomic swap transaction for `opts.schema`, given the full declared model set (to
|
|
112
|
+
* find the app→schema FKs and the schema's authz). The returned `statements` go as ONE
|
|
113
|
+
* transaction; `crossSchemaFks` and `retiring` let the command report and drive the
|
|
114
|
+
* drop-retiring / rollback lifecycle.
|
|
115
|
+
*/
|
|
116
|
+
export function renderSchemaSwap(models: ModelDescriptor[], opts: SwapOptions): SwapPlan {
|
|
117
|
+
const { schema } = opts;
|
|
118
|
+
const incoming = opts.incoming ?? `${schema}_incoming`;
|
|
119
|
+
const retiring = opts.retiring ?? `${schema}_retiring`;
|
|
120
|
+
assertSafeSchema(schema);
|
|
121
|
+
assertSafeSchema(incoming);
|
|
122
|
+
assertSafeSchema(retiring);
|
|
123
|
+
|
|
124
|
+
const fks = crossSchemaForeignKeys(models, schema);
|
|
125
|
+
// Full authz for the swapped-in schema, from scratch (live side empty = every declared grant
|
|
126
|
+
// is a create) — the incoming schema was restored with privileges stripped.
|
|
127
|
+
const statsContracts = models
|
|
128
|
+
.filter((m) => (m.schema || 'public') === schema)
|
|
129
|
+
.map((m) => compileTableContract(m));
|
|
130
|
+
const authz = emitReconcileSql({ tables: statsContracts, functions: [] }, { tables: [], functions: [] });
|
|
131
|
+
|
|
132
|
+
const statements = [
|
|
133
|
+
...fks.map((f) => f.dropSql),
|
|
134
|
+
`ALTER SCHEMA "${schema}" RENAME TO "${retiring}";`,
|
|
135
|
+
`ALTER SCHEMA "${incoming}" RENAME TO "${schema}";`,
|
|
136
|
+
...fks.map((f) => f.addSql),
|
|
137
|
+
...authz,
|
|
138
|
+
];
|
|
139
|
+
|
|
140
|
+
return { statements, crossSchemaFks: fks, retiring, incoming };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** The drop of the retiring schema, run AFTER the swap transaction commits and verify passes. */
|
|
144
|
+
export function dropRetiringSql(retiring: string): string {
|
|
145
|
+
assertSafeSchema(retiring);
|
|
146
|
+
return `DROP SCHEMA IF EXISTS "${retiring}" CASCADE;`;
|
|
147
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* search-path — the canonical introspection baseline.
|
|
3
|
+
*
|
|
4
|
+
* Every expression PostgreSQL deparses on introspection — column defaults
|
|
5
|
+
* (`pg_get_expr`), CHECK constraints, index expressions, generated columns,
|
|
6
|
+
* view/matview bodies (`pg_get_viewdef`), function bodies — renders its schema
|
|
7
|
+
* qualification RELATIVE to the session `search_path`. So the SAME schema reads
|
|
8
|
+
* differently depending on the ambient path: with a non-public schema on the
|
|
9
|
+
* path a reference to it renders BARE (`nextval('x_seq')`), off the path it
|
|
10
|
+
* renders QUALIFIED (`nextval('stats.x_seq')`). The declared state is always
|
|
11
|
+
* canonical (public bare, non-public qualified), so a non-default session or
|
|
12
|
+
* DB-level `search_path` makes every deparsed expression read as false drift.
|
|
13
|
+
*
|
|
14
|
+
* The fix is to CAPTURE under a fixed canonical path, once, at the introspection
|
|
15
|
+
* boundary — one guard for the whole class (defaults, checks, indexes, bodies),
|
|
16
|
+
* present and future. The baseline is `public`: it renders public-bare and
|
|
17
|
+
* non-public-qualified, matching the declared compile exactly (verified against
|
|
18
|
+
* PostgreSQL 16). It must be an EXPLICIT value — `RESET` / `SET … TO DEFAULT`
|
|
19
|
+
* inherit an `ALTER DATABASE/ROLE … SET search_path` override (the very thing
|
|
20
|
+
* that triggers the bug), so they are NOT canonical baselines.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import type { QueryRunner } from './authz-contract.js';
|
|
24
|
+
|
|
25
|
+
/** The canonical introspection search_path — explicit, override-proof, declared-matching. */
|
|
26
|
+
export const CANONICAL_SEARCH_PATH_SQL = 'SET search_path = public';
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Run `fn` with the connection's search_path pinned to the canonical baseline, so every
|
|
30
|
+
* expression it deparses is captured in the declared-matching form regardless of the
|
|
31
|
+
* ambient path, then RESET to the connection's default afterward. RESET restores the
|
|
32
|
+
* database/role default — which for the case this fixes (an `ALTER DATABASE … SET
|
|
33
|
+
* search_path` override) is exactly the override, so a fresh connection sees no change.
|
|
34
|
+
* It deliberately does NOT preserve a caller's transient session/transaction-local SET:
|
|
35
|
+
* restoring a captured `SET LOCAL` value session-wide would leak it (e.g. the reconcile
|
|
36
|
+
* apply's create-time wide path), and no everystack caller relies on a hand-set path
|
|
37
|
+
* surviving introspection.
|
|
38
|
+
*
|
|
39
|
+
* Effective on a persistent single connection (the direct `createUrlRunner`, `max: 1`);
|
|
40
|
+
* over a per-call pooled runner the SET does not persist, but that path already renders
|
|
41
|
+
* canonically unless the database itself carries a search_path override.
|
|
42
|
+
*/
|
|
43
|
+
export async function withCanonicalSearchPath<T>(run: QueryRunner, fn: () => Promise<T>): Promise<T> {
|
|
44
|
+
await run(CANONICAL_SEARCH_PATH_SQL);
|
|
45
|
+
try {
|
|
46
|
+
return await fn();
|
|
47
|
+
} finally {
|
|
48
|
+
// Best-effort: a failure here must not mask fn's result.
|
|
49
|
+
try { await run('RESET search_path'); } catch { /* connection may be gone */ }
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* swap-execute — the db:swap ceremony: gate → snapshot → land incoming → atomic swap → verify →
|
|
3
|
+
* drop-retiring-or-rollback, as one function over an injected runner and injected IO.
|
|
4
|
+
*
|
|
5
|
+
* The destructive moves (the S3 artifact fetch, the pg_restore into the incoming schema, the RDS
|
|
6
|
+
* snapshot, the consumer's verify validators) are all injected, so this core is the DECISION spine
|
|
7
|
+
* and is proven on a real database without S3 or a deployed stage. The command shell wires the IO.
|
|
8
|
+
*
|
|
9
|
+
* Order, none skippable (docs/plans/canonical-sync-artifacts.md):
|
|
10
|
+
* 1. FINGERPRINT GATE — the artifact's declared-schema fingerprint must equal the target's
|
|
11
|
+
* declared fingerprint, or REFUSE. Stale-shaped rows never load into a redefined schema.
|
|
12
|
+
* 2. SNAPSHOT — the rollback point, before anything destructive.
|
|
13
|
+
* 3. LAND INCOMING — restore the artifact into `<schema>_incoming` while live keeps serving.
|
|
14
|
+
* 4. ATOMIC SWAP — one transaction: drop app→schema FKs, rename out+in, recreate FKs (which
|
|
15
|
+
* RE-VALIDATE against the new data — a bad artifact rolls back here), re-apply authz. A
|
|
16
|
+
* failure rolls back whole; live never moved.
|
|
17
|
+
* 5. VERIFY — the post-commit hook runs against the swapped-in schema. A fatal verdict rolls the
|
|
18
|
+
* stage back to the snapshot; a warn surfaces without rolling back.
|
|
19
|
+
* 6. DROP RETIRING — only after verify passes.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
23
|
+
import type { QueryRunner } from './authz-contract.js';
|
|
24
|
+
import { renderSchemaSwap, dropRetiringSql } from './schema-swap.js';
|
|
25
|
+
|
|
26
|
+
/** One validator's result. A `fatal` (default) failure rolls back; a `warn` failure is surfaced only. */
|
|
27
|
+
export interface SwapCheck {
|
|
28
|
+
name: string;
|
|
29
|
+
ok: boolean;
|
|
30
|
+
detail?: string;
|
|
31
|
+
severity?: 'fatal' | 'warn';
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** The post-commit verify hook's structured verdict — names WHICH validator failed, not just "failed". */
|
|
35
|
+
export interface SwapVerdict {
|
|
36
|
+
ok: boolean;
|
|
37
|
+
checks?: SwapCheck[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface ExecuteSwapOptions {
|
|
41
|
+
models: ModelDescriptor[];
|
|
42
|
+
schema: string;
|
|
43
|
+
incoming?: string;
|
|
44
|
+
retiring?: string;
|
|
45
|
+
/** The fingerprint the artifact was stamped with (from its meta). */
|
|
46
|
+
artifactFingerprint: string;
|
|
47
|
+
/** The target's DECLARED scoped fingerprint (the caller computes it from the checkout). */
|
|
48
|
+
declaredFingerprint: string;
|
|
49
|
+
/** Restore the artifact into the incoming schema (S3 → pg_restore → schema-rewrite → psql). */
|
|
50
|
+
applyIncoming: (runner: QueryRunner) => Promise<void>;
|
|
51
|
+
/** Take the pre-swap rollback point (db:snapshot / db:backup). Optional in tests. */
|
|
52
|
+
snapshot?: () => Promise<void>;
|
|
53
|
+
/** The consumer's post-commit validators — must see committed state, so they run after COMMIT. */
|
|
54
|
+
verify?: () => Promise<SwapVerdict>;
|
|
55
|
+
/** Restore the stage to the snapshot from step 2 when verify is fatal. */
|
|
56
|
+
rollbackToSnapshot?: () => Promise<void>;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type SwapStatus =
|
|
60
|
+
| 'swapped'
|
|
61
|
+
| 'refused-fingerprint'
|
|
62
|
+
| 'refused-integrity'
|
|
63
|
+
| 'rolled-back-verify';
|
|
64
|
+
|
|
65
|
+
export interface SwapResult {
|
|
66
|
+
status: SwapStatus;
|
|
67
|
+
reason?: string;
|
|
68
|
+
/** The verify verdict, when the hook ran. */
|
|
69
|
+
verdict?: SwapVerdict;
|
|
70
|
+
/** Warn-severity checks that did NOT trigger rollback (surfaced). */
|
|
71
|
+
warnings?: SwapCheck[];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Does the verdict force a rollback? Any failing check whose severity is fatal (the default). */
|
|
75
|
+
export function isFatalVerdict(verdict: SwapVerdict): boolean {
|
|
76
|
+
if (verdict.ok) return false;
|
|
77
|
+
const checks = verdict.checks ?? [];
|
|
78
|
+
if (checks.length === 0) return true; // ok:false with no detail = fatal
|
|
79
|
+
return checks.some((c) => !c.ok && (c.severity ?? 'fatal') === 'fatal');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function executeSwap(runner: QueryRunner, opts: ExecuteSwapOptions): Promise<SwapResult> {
|
|
83
|
+
// 1. Fingerprint gate — declared-vs-declared: does the artifact and the target agree on the shape.
|
|
84
|
+
if (opts.artifactFingerprint !== opts.declaredFingerprint) {
|
|
85
|
+
return {
|
|
86
|
+
status: 'refused-fingerprint',
|
|
87
|
+
reason: `the artifact was built against ${opts.schema} fingerprint ${opts.artifactFingerprint.slice(0, 12)}, but the checkout declares ${opts.declaredFingerprint.slice(0, 12)} — rebase to the artifact's schema, or rebuild the artifact.`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const plan = renderSchemaSwap(opts.models, { schema: opts.schema, incoming: opts.incoming, retiring: opts.retiring });
|
|
92
|
+
|
|
93
|
+
// 2. Snapshot (the rollback point) before anything destructive.
|
|
94
|
+
if (opts.snapshot) await opts.snapshot();
|
|
95
|
+
|
|
96
|
+
// 3. Land the incoming schema while live keeps serving.
|
|
97
|
+
await opts.applyIncoming(runner);
|
|
98
|
+
|
|
99
|
+
// 4. The atomic swap. A FK re-validation failure (a bad artifact) rolls the whole thing back.
|
|
100
|
+
await runner('BEGIN');
|
|
101
|
+
try {
|
|
102
|
+
await runner(plan.statements.join('\n'));
|
|
103
|
+
await runner('COMMIT');
|
|
104
|
+
} catch (err: any) {
|
|
105
|
+
try { await runner('ROLLBACK'); } catch { /* already aborted */ }
|
|
106
|
+
// Clean up the landed-but-unused incoming schema so a retry is idempotent.
|
|
107
|
+
try { await runner(`DROP SCHEMA IF EXISTS "${plan.incoming}" CASCADE`); } catch { /* best effort */ }
|
|
108
|
+
return {
|
|
109
|
+
status: 'refused-integrity',
|
|
110
|
+
reason: `the swap transaction failed — usually the cross-schema FK re-validation (an app row references a ${opts.schema} key the artifact dropped): ${String(err?.message ?? err)}`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// 5. Verify (post-commit). Fatal → roll back to the snapshot; warn → surface only.
|
|
115
|
+
let verdict: SwapVerdict | undefined;
|
|
116
|
+
if (opts.verify) {
|
|
117
|
+
verdict = await opts.verify();
|
|
118
|
+
if (isFatalVerdict(verdict)) {
|
|
119
|
+
if (opts.rollbackToSnapshot) await opts.rollbackToSnapshot();
|
|
120
|
+
return {
|
|
121
|
+
status: 'rolled-back-verify',
|
|
122
|
+
reason: `verify failed: ${(verdict.checks ?? []).filter((c) => !c.ok).map((c) => c.name).join(', ') || 'no detail'} — rolled back to the pre-swap snapshot.`,
|
|
123
|
+
verdict,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// 6. Drop the retiring schema — the swap is committed and verified.
|
|
129
|
+
await runner(dropRetiringSql(plan.retiring));
|
|
130
|
+
|
|
131
|
+
const warnings = (verdict?.checks ?? []).filter((c) => !c.ok && c.severity === 'warn');
|
|
132
|
+
return { status: 'swapped', verdict, ...(warnings.length ? { warnings } : {}) };
|
|
133
|
+
}
|