@everystack/cli 0.3.22 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/package.json +9 -4
- package/src/cli/apply-authority.ts +150 -0
- package/src/cli/backfill.ts +231 -0
- package/src/cli/branch-db.ts +269 -0
- package/src/cli/commands/db-apply.ts +117 -2
- package/src/cli/commands/db-approvers.ts +71 -0
- package/src/cli/commands/db-backfill.ts +127 -0
- package/src/cli/commands/db-branch.ts +220 -0
- package/src/cli/commands/db-check.ts +16 -58
- package/src/cli/commands/db-plan.ts +21 -0
- package/src/cli/commands/db-sync.ts +26 -0
- package/src/cli/db-build.ts +161 -0
- package/src/cli/derived-source.ts +3 -2
- package/src/cli/edge-plan.ts +35 -8
- package/src/cli/index.ts +20 -1
- package/src/cli/model-api.ts +7 -0
- package/src/cli/schema-source.ts +5 -0
- package/src/cli/state-apply.ts +30 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `everystack db:template:refresh` + `everystack db:branch` — per-branch
|
|
3
|
+
* dev databases from a models-built template (brick 5-remainder,
|
|
4
|
+
* decision 14).
|
|
5
|
+
*
|
|
6
|
+
* db:template:refresh [--database-url <url>] [--models <barrel>]
|
|
7
|
+
* [--sql-dir db/sql] [--seed "<cmd>"] [--no-seed]
|
|
8
|
+
* db:branch # mint (or find) the current branch's DB
|
|
9
|
+
* db:branch --list # every branch DB, mapped to its branch
|
|
10
|
+
* db:branch --drop --confirm # drop the current branch's DB
|
|
11
|
+
* db:branch --prune --confirm # drop DBs whose git branch is gone
|
|
12
|
+
*
|
|
13
|
+
* The template is built FROM THE DECLARED STATE (both layers, fingerprint
|
|
14
|
+
* MATCH bar) plus seed-as-code — the app's own `db:seed` script, run as a
|
|
15
|
+
* subprocess with DATABASE_URL pointed at the fresh template. Never a data
|
|
16
|
+
* copy: deterministic, PII-free (real data enters branch testing through
|
|
17
|
+
* stage forks, brick 10). Branch databases mint via `CREATE DATABASE …
|
|
18
|
+
* TEMPLATE`, keyed on the git branch, and `db:sync` evolves them with the
|
|
19
|
+
* checkout — disposable by construction, never long-lived, never poisoned.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import fs from 'node:fs/promises';
|
|
23
|
+
import { spawnSync } from 'node:child_process';
|
|
24
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
25
|
+
import {
|
|
26
|
+
executeTemplateRefresh,
|
|
27
|
+
ensureBranchDatabase,
|
|
28
|
+
listBranchDatabases,
|
|
29
|
+
readTemplateProvenance,
|
|
30
|
+
prunableBranchDatabases,
|
|
31
|
+
dropBranchDatabase,
|
|
32
|
+
branchDatabaseName,
|
|
33
|
+
baseNameFromUrl,
|
|
34
|
+
templateName,
|
|
35
|
+
currentBranch,
|
|
36
|
+
localBranches,
|
|
37
|
+
} from '../branch-db.js';
|
|
38
|
+
import { currentGitRef } from '../state-apply.js';
|
|
39
|
+
import { fingerprintModels } from '../schema-fingerprint.js';
|
|
40
|
+
import { resolveDbSource, type DbSource } from '../db-source.js';
|
|
41
|
+
import { resolveModelsPath } from '../models-path.js';
|
|
42
|
+
import { readSqlDirIfPresent } from './db-sync.js';
|
|
43
|
+
import { loadModels } from './db-generate.js';
|
|
44
|
+
import { step, success, fail, info, warn } from '../output.js';
|
|
45
|
+
|
|
46
|
+
function requireUrl(flags: Record<string, string>, verb: string): string {
|
|
47
|
+
let dbSource: DbSource;
|
|
48
|
+
try {
|
|
49
|
+
dbSource = resolveDbSource(flags);
|
|
50
|
+
} catch (err: any) {
|
|
51
|
+
fail(err.message);
|
|
52
|
+
process.exit(1);
|
|
53
|
+
}
|
|
54
|
+
if (dbSource.kind !== 'url') {
|
|
55
|
+
fail(`${verb} manages local databases — it needs a direct connection (--database-url or DATABASE_URL).`);
|
|
56
|
+
process.exit(1);
|
|
57
|
+
}
|
|
58
|
+
return dbSource.url;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The seed convention: the app's own `db:seed` script from package.json. */
|
|
62
|
+
async function detectSeedCommand(): Promise<string | null> {
|
|
63
|
+
try {
|
|
64
|
+
const pkg = JSON.parse(await fs.readFile('package.json', 'utf8'));
|
|
65
|
+
return pkg?.scripts?.['db:seed'] ? 'npm run db:seed' : null;
|
|
66
|
+
} catch {
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function runSeedSubprocess(command: string, templateUrl: string): void {
|
|
72
|
+
const result = spawnSync(command, {
|
|
73
|
+
shell: true,
|
|
74
|
+
stdio: 'inherit',
|
|
75
|
+
env: { ...process.env, DATABASE_URL: templateUrl },
|
|
76
|
+
});
|
|
77
|
+
if (result.status !== 0) {
|
|
78
|
+
throw new Error(`seed command failed (exit ${result.status}): ${command}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export async function dbTemplateRefreshCommand(flags: Record<string, string>): Promise<void> {
|
|
83
|
+
const url = requireUrl(flags, 'db:template:refresh');
|
|
84
|
+
|
|
85
|
+
const modelsPath = resolveModelsPath(flags.models);
|
|
86
|
+
let models: ModelDescriptor[];
|
|
87
|
+
try {
|
|
88
|
+
step(`Loading models from ${modelsPath}...`);
|
|
89
|
+
models = await loadModels(modelsPath);
|
|
90
|
+
info(`${models.length} model(s).`);
|
|
91
|
+
} catch (err: any) {
|
|
92
|
+
fail(err.message);
|
|
93
|
+
process.exit(1);
|
|
94
|
+
}
|
|
95
|
+
const sources = await readSqlDirIfPresent(flags['sql-dir'] || 'db/sql');
|
|
96
|
+
|
|
97
|
+
let seedCommand: string | null = null;
|
|
98
|
+
if (flags['no-seed'] === 'true') {
|
|
99
|
+
info('Seeding skipped (--no-seed).');
|
|
100
|
+
} else if (flags.seed && flags.seed !== 'true') {
|
|
101
|
+
seedCommand = flags.seed;
|
|
102
|
+
} else {
|
|
103
|
+
seedCommand = await detectSeedCommand();
|
|
104
|
+
if (seedCommand) info(`Seed: package.json db:seed script (override with --seed "<cmd>" / --no-seed).`);
|
|
105
|
+
else warn('No db:seed script in package.json — the template builds UNSEEDED (declare one, or pass --seed "<cmd>").');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const template = templateName(baseNameFromUrl(url));
|
|
109
|
+
step(`Rebuilding ${template} from the declared state (drop → create → sync both layers → seed)...`);
|
|
110
|
+
try {
|
|
111
|
+
const result = await executeTemplateRefresh(url, models, {
|
|
112
|
+
sources,
|
|
113
|
+
actor: process.env.USER ?? null,
|
|
114
|
+
gitRef: currentGitRef(),
|
|
115
|
+
seed: seedCommand === null ? null : (templateUrl) => {
|
|
116
|
+
step(`Seeding: ${seedCommand} (DATABASE_URL → ${redactUrl(templateUrl)})...`);
|
|
117
|
+
runSeedSubprocess(seedCommand!, templateUrl);
|
|
118
|
+
return Promise.resolve();
|
|
119
|
+
},
|
|
120
|
+
});
|
|
121
|
+
for (const line of result.report) info(` ${line}`);
|
|
122
|
+
success(`Template ${result.database} is the declared state at ${result.fingerprint.slice(0, 12)}${seedCommand ? ', seeded' : ''}. Mint branch databases: everystack db:branch`);
|
|
123
|
+
} catch (err: any) {
|
|
124
|
+
fail(`Template refresh failed — no template was left behind (all or nothing): ${err.message}`);
|
|
125
|
+
process.exit(1);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/** Redact credentials when echoing a URL. */
|
|
130
|
+
function redactUrl(url: string): string {
|
|
131
|
+
return url.replace(/\/\/[^@/]+@/, '//***@');
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export async function dbBranchCommand(flags: Record<string, string>): Promise<void> {
|
|
135
|
+
const url = requireUrl(flags, 'db:branch');
|
|
136
|
+
|
|
137
|
+
if (flags.list === 'true') {
|
|
138
|
+
const provenance = await readTemplateProvenance(url);
|
|
139
|
+
if (provenance) {
|
|
140
|
+
info(`template ${templateName(baseNameFromUrl(url))}: fingerprint ${provenance.fingerprint.slice(0, 12)}, built ${provenance.builtAt}${provenance.gitRef ? ` at ${provenance.gitRef.slice(0, 12)}` : ''}`);
|
|
141
|
+
} else {
|
|
142
|
+
warn(`no template — build it once: everystack db:template:refresh`);
|
|
143
|
+
}
|
|
144
|
+
const all = await listBranchDatabases(url);
|
|
145
|
+
if (all.length === 0) {
|
|
146
|
+
info('no branch databases.');
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const live = new Set(localBranches());
|
|
150
|
+
for (const d of all) {
|
|
151
|
+
const state = d.branch === null ? 'branch unknown (comment lost)' : live.has(d.branch) ? d.branch : `${d.branch} — branch GONE (db:branch --prune)`;
|
|
152
|
+
info(` ${d.database} ← ${state}`);
|
|
153
|
+
}
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
if (flags.prune === 'true') {
|
|
158
|
+
const dead = prunableBranchDatabases(await listBranchDatabases(url), localBranches());
|
|
159
|
+
if (dead.length === 0) {
|
|
160
|
+
success('Nothing to prune — every branch database maps to a live branch.');
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
for (const d of dead) info(` ${d.database} ← ${d.branch} (gone)`);
|
|
164
|
+
if (flags.confirm !== 'true') {
|
|
165
|
+
fail(`Would drop the ${dead.length} database(s) above — confirm with --confirm.`);
|
|
166
|
+
process.exit(1);
|
|
167
|
+
}
|
|
168
|
+
for (const d of dead) {
|
|
169
|
+
await dropBranchDatabase(url, d.database);
|
|
170
|
+
info(`dropped ${d.database}`);
|
|
171
|
+
}
|
|
172
|
+
success(`Pruned ${dead.length} branch database(s).`);
|
|
173
|
+
return;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const branch = currentBranch();
|
|
177
|
+
if (branch === null) {
|
|
178
|
+
fail('db:branch is keyed on the git branch — not a repository, or detached HEAD.');
|
|
179
|
+
process.exit(1);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
if (flags.drop === 'true') {
|
|
183
|
+
const database = branchDatabaseName(baseNameFromUrl(url), branch);
|
|
184
|
+
if (flags.confirm !== 'true') {
|
|
185
|
+
fail(`Would drop ${database} (branch ${branch}) — confirm with --confirm.`);
|
|
186
|
+
process.exit(1);
|
|
187
|
+
}
|
|
188
|
+
await dropBranchDatabase(url, database);
|
|
189
|
+
success(`Dropped ${database}.`);
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
const result = await ensureBranchDatabase(url, branch);
|
|
194
|
+
switch (result.status) {
|
|
195
|
+
case 'no-template':
|
|
196
|
+
fail(`No template ${result.template} — build it once: everystack db:template:refresh. Branch databases mint from it.`);
|
|
197
|
+
process.exit(1);
|
|
198
|
+
break;
|
|
199
|
+
case 'exists':
|
|
200
|
+
info(`${result.database} already exists for branch ${branch}.`);
|
|
201
|
+
break;
|
|
202
|
+
case 'created': {
|
|
203
|
+
success(`Minted ${result.database} from ${result.template} (branch ${branch}).`);
|
|
204
|
+
try {
|
|
205
|
+
const provenance = await readTemplateProvenance(url);
|
|
206
|
+
const declared = fingerprintModels(await loadModels(resolveModelsPath(flags.models))).hash;
|
|
207
|
+
if (provenance && provenance.fingerprint !== declared) {
|
|
208
|
+
info(`· your checkout declares ${declared.slice(0, 12)}; the template was built at ${provenance.fingerprint.slice(0, 12)} — db:sync below evolves it (refresh the template when it drifts far).`);
|
|
209
|
+
}
|
|
210
|
+
} catch {
|
|
211
|
+
// The staleness note is best-effort — minting already succeeded.
|
|
212
|
+
}
|
|
213
|
+
break;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
console.log('');
|
|
217
|
+
console.log(` export DATABASE_URL="${result.url}"`);
|
|
218
|
+
console.log('');
|
|
219
|
+
info(`then: everystack db:sync # evolve it with your checkout as you edit`);
|
|
220
|
+
}
|
|
@@ -28,20 +28,22 @@
|
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
30
|
import fs from 'node:fs/promises';
|
|
31
|
-
import postgres from 'postgres';
|
|
32
31
|
import type { ModelDescriptor } from '@everystack/model';
|
|
33
|
-
import type { TableContract } from '../authz-contract.js';
|
|
34
32
|
import { compileDeclaredState } from '../declared-diff.js';
|
|
35
33
|
import { compileMigration } from '../migration-compile.js';
|
|
36
34
|
import { compileDrizzleSource } from '../schema-source.js';
|
|
37
35
|
import { parseSqlSources, type SourceFile } from '../derived-source.js';
|
|
38
36
|
import { currentGitRef } from '../state-apply.js';
|
|
39
|
-
import {
|
|
37
|
+
import { createDatabase, dropDatabase, buildIntoDatabase, withDatabase } from '../db-build.js';
|
|
40
38
|
import { resolveModelsPath } from '../models-path.js';
|
|
41
|
-
import {
|
|
39
|
+
import { readSqlDirIfPresent } from './db-sync.js';
|
|
42
40
|
import { loadModels } from './db-generate.js';
|
|
43
41
|
import { step, success, fail, info, warn } from '../output.js';
|
|
44
42
|
|
|
43
|
+
// The role derivation moved to the shared builder (db-build) with brick
|
|
44
|
+
// 5-remainder; re-exported here because db:check is where it grew up.
|
|
45
|
+
export { rolesFromContract } from '../db-build.js';
|
|
46
|
+
|
|
45
47
|
const DEFAULT_SQL_DIR = 'db/sql';
|
|
46
48
|
const DEFAULT_SCHEMA_OUT = 'db/schema.generated.ts';
|
|
47
49
|
|
|
@@ -145,19 +147,6 @@ export function checkFailed(findings: CheckFinding[]): boolean {
|
|
|
145
147
|
// The ephemeral compose ring.
|
|
146
148
|
// ---------------------------------------------------------------------------
|
|
147
149
|
|
|
148
|
-
/** Every role the contract grants to or names in a policy — the compose DB needs them. */
|
|
149
|
-
export function rolesFromContract(tables: TableContract[]): string[] {
|
|
150
|
-
const roles = new Set<string>();
|
|
151
|
-
for (const t of tables) {
|
|
152
|
-
for (const grantee of Object.keys(t.grants)) roles.add(grantee);
|
|
153
|
-
for (const grantee of Object.keys(t.columnGrants ?? {})) roles.add(grantee);
|
|
154
|
-
for (const p of t.policies) for (const r of p.roles) roles.add(r);
|
|
155
|
-
}
|
|
156
|
-
roles.delete('PUBLIC');
|
|
157
|
-
roles.delete('public');
|
|
158
|
-
return [...roles].sort();
|
|
159
|
-
}
|
|
160
|
-
|
|
161
150
|
export interface EphemeralComposeResult {
|
|
162
151
|
database: string;
|
|
163
152
|
createdRoles: string[];
|
|
@@ -168,12 +157,10 @@ export interface EphemeralComposeResult {
|
|
|
168
157
|
}
|
|
169
158
|
|
|
170
159
|
/**
|
|
171
|
-
* Build the declared state from scratch on an ephemeral database
|
|
172
|
-
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
* fresh database has nothing unmodeled and nothing to drop, so anything
|
|
176
|
-
* short of MATCH is a real compose fault.
|
|
160
|
+
* Build the declared state from scratch on an ephemeral database (the
|
|
161
|
+
* shared db-build core), then DROP it — always, error or not. The bar is
|
|
162
|
+
* `converged && fingerprintMatch`: a fresh database has nothing unmodeled
|
|
163
|
+
* and nothing to drop, so anything short of MATCH is a real compose fault.
|
|
177
164
|
*/
|
|
178
165
|
export async function executeEphemeralCompose(
|
|
179
166
|
adminUrl: string,
|
|
@@ -181,46 +168,17 @@ export async function executeEphemeralCompose(
|
|
|
181
168
|
sources: SourceFile[] | null,
|
|
182
169
|
opts: { actor?: string | null; gitRef?: string | null } = {},
|
|
183
170
|
): Promise<EphemeralComposeResult> {
|
|
184
|
-
const
|
|
185
|
-
|
|
186
|
-
try {
|
|
187
|
-
await admin.unsafe(`CREATE DATABASE "${dbName}"`);
|
|
188
|
-
} finally {
|
|
189
|
-
await admin.end();
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const { runner, end } = await createUrlRunner(adminUrl.replace(/\/[^/?]+(\?|$)/, `/${dbName}$1`));
|
|
171
|
+
const database = `escheck_${process.pid}_${Date.now()}`;
|
|
172
|
+
await createDatabase(adminUrl, database);
|
|
193
173
|
try {
|
|
194
|
-
const
|
|
195
|
-
|
|
196
|
-
if (!/^[a-z_][a-z0-9_$]*$/i.test(role)) {
|
|
197
|
-
throw new Error(`contract references a role name that is not a plain identifier: ${JSON.stringify(role)}`);
|
|
198
|
-
}
|
|
199
|
-
await runner(
|
|
200
|
-
`DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN CREATE ROLE "${role}" NOLOGIN; END IF; END $$`,
|
|
201
|
-
);
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
const run = await executeSync(runner, models, sources, {
|
|
174
|
+
const built = await buildIntoDatabase(withDatabase(adminUrl, database), models, {
|
|
175
|
+
sources,
|
|
205
176
|
actor: opts.actor ?? 'db:check',
|
|
206
177
|
gitRef: opts.gitRef ?? null,
|
|
207
178
|
});
|
|
208
|
-
return {
|
|
209
|
-
database: dbName,
|
|
210
|
-
createdRoles: roles,
|
|
211
|
-
converged: run.converged,
|
|
212
|
-
fingerprintMatch: run.fingerprintMatch,
|
|
213
|
-
fingerprint: run.state.toFingerprint,
|
|
214
|
-
report: buildSyncReport(run),
|
|
215
|
-
};
|
|
179
|
+
return { database, ...built };
|
|
216
180
|
} finally {
|
|
217
|
-
await
|
|
218
|
-
const reaper = postgres(adminUrl, { max: 1 });
|
|
219
|
-
try {
|
|
220
|
-
await reaper.unsafe(`DROP DATABASE IF EXISTS "${dbName}" WITH (FORCE)`);
|
|
221
|
-
} finally {
|
|
222
|
-
await reaper.end();
|
|
223
|
-
}
|
|
181
|
+
await dropDatabase(adminUrl, database);
|
|
224
182
|
}
|
|
225
183
|
}
|
|
226
184
|
|
|
@@ -25,6 +25,8 @@ import { introspectSchema } from '../schema-introspect.js';
|
|
|
25
25
|
import { FUNCTIONS_SQL, contractFunctionRow } from '../security-catalog.js';
|
|
26
26
|
import { mintEdgePlan, planHash, buildPlanSummary } from '../edge-plan.js';
|
|
27
27
|
import { verifyDescent } from '../git-descent.js';
|
|
28
|
+
import { planBackfills, readBackfillLog } from '../backfill.js';
|
|
29
|
+
import { readSqlDirIfPresent } from './db-sync.js';
|
|
28
30
|
import { currentGitRef } from '../state-apply.js';
|
|
29
31
|
import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
|
|
30
32
|
import { resolveModelsPath } from '../models-path.js';
|
|
@@ -113,6 +115,25 @@ export async function dbPlanCommand(flags: Record<string, string>): Promise<void
|
|
|
113
115
|
warn('db:apply from this checkout will refuse this plan — rebase first, then re-mint.');
|
|
114
116
|
}
|
|
115
117
|
}
|
|
118
|
+
|
|
119
|
+
// Visibility, never automation (brick 8): the review surface names the
|
|
120
|
+
// target's pending backfills — expand → backfill → contract stays a
|
|
121
|
+
// sequence of deliberate acts, and a contraction plan minted while its
|
|
122
|
+
// data move is unrun should be caught by the human reading this.
|
|
123
|
+
const backfillFiles = await readSqlDirIfPresent('db/backfills');
|
|
124
|
+
if (backfillFiles !== null) {
|
|
125
|
+
try {
|
|
126
|
+
const bf = planBackfills(backfillFiles, await readBackfillLog(runner));
|
|
127
|
+
if (bf.blocked.length > 0) {
|
|
128
|
+
warn(`${bf.blocked.length} backfill(s) BLOCKED on this target (edited after running) — everystack db:backfill names the fix.`);
|
|
129
|
+
}
|
|
130
|
+
if (bf.pending.length > 0) {
|
|
131
|
+
warn(`${bf.pending.length} backfill(s) pending on this target: ${bf.pending.map((p) => p.file).join(', ')} — run them deliberately (db:backfill --apply) before a contraction lands.`);
|
|
132
|
+
}
|
|
133
|
+
} catch (err: any) {
|
|
134
|
+
warn(`backfill lane: could not read everystack.backfill_log (${err.message}) — pending-backfill visibility unavailable.`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
116
137
|
if (out !== '-') {
|
|
117
138
|
success(`Wrote ${out} — review it, then \`everystack db:apply --plan ${out}\`.`);
|
|
118
139
|
warn('Plans are ephemeral release artifacts — attach to the run, do NOT commit.');
|
|
@@ -37,6 +37,7 @@ import { generateMigrationSql, unmodeledTables } from '../migration-generate.js'
|
|
|
37
37
|
import {
|
|
38
38
|
applyStateAndVerify,
|
|
39
39
|
classifyGeneratedStatements,
|
|
40
|
+
classifyDestructive,
|
|
40
41
|
currentGitRef,
|
|
41
42
|
type ClassifiedStatements,
|
|
42
43
|
type StateSyncOutcome,
|
|
@@ -45,6 +46,7 @@ import { fingerprintModels } from '../schema-fingerprint.js';
|
|
|
45
46
|
import { compileDrizzleSource } from '../schema-source.js';
|
|
46
47
|
import type { SourceFile } from '../derived-source.js';
|
|
47
48
|
import { resolveDbSource, createUrlRunner, type DbSource } from '../db-source.js';
|
|
49
|
+
import { planBackfills, readBackfillLog } from '../backfill.js';
|
|
48
50
|
import { resolveModelsPath } from '../models-path.js';
|
|
49
51
|
import { executeReconcile, buildReconcileReport, type ReconcileRun } from './db-reconcile.js';
|
|
50
52
|
import { loadModels } from './db-generate.js';
|
|
@@ -277,6 +279,10 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
277
279
|
onStatePlan: (statements, classified) => {
|
|
278
280
|
if (classified.executable.length > 0) {
|
|
279
281
|
info(`state plan — ${classified.executable.length} statement(s):`);
|
|
282
|
+
const { drops, narrowings } = classifyDestructive(classified.executable);
|
|
283
|
+
if (drops.length + narrowings.length > 0) {
|
|
284
|
+
warn(`${drops.length + narrowings.length} DESTRUCTIVE — ${drops.length} drop(s), ${narrowings.length} narrowing type change(s). Dev sync runs them; protected stages gate them (db:plan → db:apply).`);
|
|
285
|
+
}
|
|
280
286
|
console.log('');
|
|
281
287
|
for (const s of statements) console.log(s.endsWith(';') ? s : `${s};`);
|
|
282
288
|
console.log('');
|
|
@@ -289,8 +295,22 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
289
295
|
},
|
|
290
296
|
);
|
|
291
297
|
|
|
298
|
+
// Visibility, never automation (brick 8): pending backfills surface
|
|
299
|
+
// here, but a schema verb never runs a data job.
|
|
300
|
+
let backfills: { pending: number; blocked: number } | null = null;
|
|
301
|
+
const backfillFiles = await readSqlDirIfPresent('db/backfills');
|
|
302
|
+
if (backfillFiles !== null) {
|
|
303
|
+
try {
|
|
304
|
+
const bf = planBackfills(backfillFiles, await readBackfillLog(runner));
|
|
305
|
+
backfills = { pending: bf.pending.length, blocked: bf.blocked.length };
|
|
306
|
+
} catch (err: any) {
|
|
307
|
+
warn(`backfill lane: could not read everystack.backfill_log (${err.message}) — pending-backfill visibility unavailable.`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
292
311
|
if (flags.json === 'true') {
|
|
293
312
|
console.log(JSON.stringify({
|
|
313
|
+
backfills,
|
|
294
314
|
state: {
|
|
295
315
|
applied: run.state.applied,
|
|
296
316
|
executed: run.state.classified.executable.length,
|
|
@@ -312,6 +332,12 @@ export async function dbSyncCommand(flags: Record<string, string>): Promise<void
|
|
|
312
332
|
}, null, 2));
|
|
313
333
|
} else {
|
|
314
334
|
for (const line of buildSyncReport(run)) info(line);
|
|
335
|
+
if (backfills && backfills.blocked > 0) {
|
|
336
|
+
warn(`${backfills.blocked} backfill(s) BLOCKED (edited after running) — everystack db:backfill names the fix.`);
|
|
337
|
+
}
|
|
338
|
+
if (backfills && backfills.pending > 0) {
|
|
339
|
+
info(`· ${backfills.pending} backfill(s) pending — data jobs run deliberately, never as a sync side effect: everystack db:backfill --apply`);
|
|
340
|
+
}
|
|
315
341
|
console.log('');
|
|
316
342
|
if (!run.converged) {
|
|
317
343
|
fail('NOT converged — fix the findings above and re-run db:sync.');
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* db-build — build a database FROM the declared state (brick 5-remainder's
|
|
3
|
+
* shared core; decision 14: models-built, never data copies).
|
|
4
|
+
*
|
|
5
|
+
* One builder, three consumers: `db:check`'s ephemeral compose ring, the
|
|
6
|
+
* dev template (`db:template:refresh`), and `createEphemeralDatabase` — the
|
|
7
|
+
* exported test-database helper (the create-sync-drop shape this repo's own
|
|
8
|
+
* integration harness proves on every run, packaged for consumers). The bar
|
|
9
|
+
* is always the brick-2 identity: a fresh database built from the declared
|
|
10
|
+
* state lands EXACTLY on the models' fingerprint.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { ModelDescriptor } from '@everystack/model';
|
|
14
|
+
import type { TableContract, QueryRunner } from './authz-contract.js';
|
|
15
|
+
import type { SourceFile } from './derived-source.js';
|
|
16
|
+
import { compileDeclaredState } from './declared-diff.js';
|
|
17
|
+
import { createUrlRunner } from './db-source.js';
|
|
18
|
+
import { executeSync, buildSyncReport } from './commands/db-sync.js';
|
|
19
|
+
|
|
20
|
+
const SAFE_NAME = /^[a-z_][a-z0-9_$]*$/;
|
|
21
|
+
|
|
22
|
+
function assertSafeName(name: string, what: string): void {
|
|
23
|
+
if (!SAFE_NAME.test(name)) {
|
|
24
|
+
throw new Error(`${what} ${JSON.stringify(name)} is not a plain lowercase identifier — everystack manages databases it can name without quoting games.`);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Swap the database path segment of a connection URL. */
|
|
29
|
+
export function withDatabase(url: string, database: string): string {
|
|
30
|
+
if (!/\/[^/?]+(\?|$)/.test(url)) {
|
|
31
|
+
throw new Error('the connection URL names no database — add the path segment (postgres://…/mydb)');
|
|
32
|
+
}
|
|
33
|
+
return url.replace(/\/[^/?]+(\?|$)/, `/${database}$1`);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Every role the contract grants to or names in a policy — a fresh database needs them. */
|
|
37
|
+
export function rolesFromContract(tables: TableContract[]): string[] {
|
|
38
|
+
const roles = new Set<string>();
|
|
39
|
+
for (const t of tables) {
|
|
40
|
+
for (const grantee of Object.keys(t.grants)) roles.add(grantee);
|
|
41
|
+
for (const grantee of Object.keys(t.columnGrants ?? {})) roles.add(grantee);
|
|
42
|
+
for (const p of t.policies) for (const r of p.roles) roles.add(r);
|
|
43
|
+
}
|
|
44
|
+
roles.delete('PUBLIC');
|
|
45
|
+
roles.delete('public');
|
|
46
|
+
return [...roles].sort();
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Create the contract's roles, NOLOGIN, only if absent (roles are cluster-level). */
|
|
50
|
+
export async function ensureContractRoles(runner: QueryRunner, models: ModelDescriptor[]): Promise<string[]> {
|
|
51
|
+
const roles = rolesFromContract(compileDeclaredState(models).contract.tables);
|
|
52
|
+
for (const role of roles) {
|
|
53
|
+
assertSafeName(role, 'contract role');
|
|
54
|
+
await runner(
|
|
55
|
+
`DO $$ BEGIN IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN CREATE ROLE "${role}" NOLOGIN; END IF; END $$`,
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
return roles;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function createDatabase(adminUrl: string, name: string, template?: string): Promise<void> {
|
|
62
|
+
assertSafeName(name, 'database name');
|
|
63
|
+
if (template !== undefined) assertSafeName(template, 'template name');
|
|
64
|
+
const { runner, end } = await createUrlRunner(adminUrl);
|
|
65
|
+
try {
|
|
66
|
+
await runner(`CREATE DATABASE "${name}"${template ? ` TEMPLATE "${template}"` : ''}`);
|
|
67
|
+
} finally {
|
|
68
|
+
await end?.();
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export async function dropDatabase(adminUrl: string, name: string): Promise<void> {
|
|
73
|
+
assertSafeName(name, 'database name');
|
|
74
|
+
const { runner, end } = await createUrlRunner(adminUrl);
|
|
75
|
+
try {
|
|
76
|
+
await runner(`DROP DATABASE IF EXISTS "${name}" WITH (FORCE)`);
|
|
77
|
+
} finally {
|
|
78
|
+
await end?.();
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface BuildResult {
|
|
83
|
+
converged: boolean;
|
|
84
|
+
fingerprintMatch: boolean;
|
|
85
|
+
fingerprint: string;
|
|
86
|
+
createdRoles: string[];
|
|
87
|
+
report: string[];
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export interface BuildOptions {
|
|
91
|
+
sources?: SourceFile[] | null;
|
|
92
|
+
actor?: string | null;
|
|
93
|
+
gitRef?: string | null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Sync a (typically fresh) database to the declared state — both layers —
|
|
98
|
+
* and report against the MATCH bar. Opens and closes its own connection,
|
|
99
|
+
* so the database is immediately usable as a `TEMPLATE` source afterwards.
|
|
100
|
+
*/
|
|
101
|
+
export async function buildIntoDatabase(
|
|
102
|
+
url: string,
|
|
103
|
+
models: ModelDescriptor[],
|
|
104
|
+
options: BuildOptions = {},
|
|
105
|
+
): Promise<BuildResult> {
|
|
106
|
+
const { runner, end } = await createUrlRunner(url);
|
|
107
|
+
try {
|
|
108
|
+
const createdRoles = await ensureContractRoles(runner, models);
|
|
109
|
+
const run = await executeSync(runner, models, options.sources ?? null, {
|
|
110
|
+
actor: options.actor ?? 'db-build',
|
|
111
|
+
gitRef: options.gitRef ?? null,
|
|
112
|
+
});
|
|
113
|
+
return {
|
|
114
|
+
converged: run.converged,
|
|
115
|
+
fingerprintMatch: run.fingerprintMatch,
|
|
116
|
+
fingerprint: run.state.toFingerprint,
|
|
117
|
+
createdRoles,
|
|
118
|
+
report: buildSyncReport(run),
|
|
119
|
+
};
|
|
120
|
+
} finally {
|
|
121
|
+
await end?.();
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export interface EphemeralDatabase {
|
|
126
|
+
database: string;
|
|
127
|
+
url: string;
|
|
128
|
+
fingerprint: string;
|
|
129
|
+
drop: () => Promise<void>;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* The packaged create-sync-drop shape for consumer test suites: a fresh
|
|
134
|
+
* database at the declared state (state + authz + derived layer), never
|
|
135
|
+
* long-lived, so never poisoned. Throws (and drops) unless the build lands
|
|
136
|
+
* MATCH — a test database that isn't the declared state proves nothing.
|
|
137
|
+
*/
|
|
138
|
+
export async function createEphemeralDatabase(
|
|
139
|
+
adminUrl: string,
|
|
140
|
+
models: ModelDescriptor[],
|
|
141
|
+
options: BuildOptions & { name?: string } = {},
|
|
142
|
+
): Promise<EphemeralDatabase> {
|
|
143
|
+
const database = options.name ?? `eph_${process.pid}_${Date.now()}`;
|
|
144
|
+
await createDatabase(adminUrl, database);
|
|
145
|
+
const url = withDatabase(adminUrl, database);
|
|
146
|
+
try {
|
|
147
|
+
const built = await buildIntoDatabase(url, models, options);
|
|
148
|
+
if (!(built.converged && built.fingerprintMatch)) {
|
|
149
|
+
throw new Error(`the ephemeral database did not land on the declared state:\n${built.report.join('\n')}`);
|
|
150
|
+
}
|
|
151
|
+
return {
|
|
152
|
+
database,
|
|
153
|
+
url,
|
|
154
|
+
fingerprint: built.fingerprint,
|
|
155
|
+
drop: () => dropDatabase(adminUrl, database),
|
|
156
|
+
};
|
|
157
|
+
} catch (err) {
|
|
158
|
+
await dropDatabase(adminUrl, database).catch(() => {});
|
|
159
|
+
throw err;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
@@ -62,7 +62,7 @@ export interface ParsedSources {
|
|
|
62
62
|
// Lexer.
|
|
63
63
|
// ---------------------------------------------------------------------------
|
|
64
64
|
|
|
65
|
-
type LexState =
|
|
65
|
+
export type LexState =
|
|
66
66
|
| { in: 'normal' }
|
|
67
67
|
| { in: 'single'; escaped: boolean }
|
|
68
68
|
| { in: 'double' }
|
|
@@ -82,7 +82,8 @@ function dollarTagAt(text: string, i: number): string | null {
|
|
|
82
82
|
* (comments/strings/normal). The shared core under both the statement splitter
|
|
83
83
|
* and the normalizer, so they can never disagree about where a string ends.
|
|
84
84
|
*/
|
|
85
|
-
|
|
85
|
+
/** The dollar-quote-aware SQL lexer — exported for siblings that need string-safe scans (the backfill lane). */
|
|
86
|
+
export function lex(
|
|
86
87
|
text: string,
|
|
87
88
|
emit: (ch: string, state: LexState['in'], index: number) => void,
|
|
88
89
|
): void {
|